pinokiod 7.3.10 → 7.3.11

package/kernel/bin/brew.js

@@ -51,7 +51,11 @@ class Brew {
     await this.kernel.bin.rm("Homebrew.zip", ondata)
 //
     ondata({ raw: "installing gettext\r\n" })
-    await this.kernel.bin.exec({ message: "brew install gettext --force-bottle", conda: { skip: true } }, (stream) => { ondata(stream) })
+    await this.kernel.bin.exec({
+      message: "brew install gettext --force-bottle",
+      conda: { skip: true },
+      env: this.nonInteractiveEnv()
+    }, (stream) => { ondata(stream) })
 //
     ondata({ raw: `Install finished\r\n` })
   }
@@ -96,7 +100,13 @@ class Brew {
       HOMEBREW_PREFIX: this.kernel.bin.path("homebrew"),
       HOMEBREW_CELLAR: this.kernel.bin.path("homebrew", "Cellar"),
       HOMEBREW_REPOSITORY: this.kernel.bin.path("homebrew"),
-      HOMEBREW_CACHE: this.kernel.bin.path("homebrew", "cache")
+      HOMEBREW_CACHE: this.kernel.bin.path("homebrew", "cache"),
+      ...this.nonInteractiveEnv()
+    }
+  }
+  nonInteractiveEnv() {
+    return {
+      HOMEBREW_NO_ASK: "1"
     }
   }
 }

package/kernel/bin/caddy.js

@@ -5,6 +5,12 @@ const semver = require('semver')
 const kill = require('kill-sync')
 const Util = require('../util')
 
+const ADMIN_READY_TIMEOUT_MS = 30000
+const ADMIN_START_TIMEOUT_MS = 5000
+const ADMIN_POLL_INTERVAL_MS = 250
+
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms))
+
 class Caddy {
   description = "Installs Caddy for automatic local HTTPS and network sharing. You may be asked to approve certificate setup."
   cmd() {
@@ -104,17 +110,7 @@ class Caddy {
           }
         })
       })
-      let admin_running = false
-      const admin_wait_started = Date.now()
-      while ((Date.now() - admin_wait_started) < 5000) {
-        admin_running = await this.running()
-        if (admin_running) {
-          break
-        }
-        await new Promise((resolve) => {
-          setTimeout(resolve, 250)
-        })
-      }
+      const admin_running = await this.waitForAdmin(ADMIN_START_TIMEOUT_MS)
       if (!admin_running) {
         console.log("caddy admin unavailable after startup")
         if (startup_output.length > 0) {
@@ -132,6 +128,16 @@ class Caddy {
 //      this.kernel.refresh(true)
     }
   }
+  async waitForAdmin(timeout = ADMIN_READY_TIMEOUT_MS, interval = ADMIN_POLL_INTERVAL_MS) {
+    const started = Date.now()
+    while ((Date.now() - started) < timeout) {
+      if (await this.running()) {
+        return true
+      }
+      await sleep(interval)
+    }
+    return false
+  }
   async install(req, ondata, kernel, id) {
 //    let fullpath = path.resolve(kernel.homedir, "ENVIRONMENT")
 //    await Util.update_env(fullpath, {
@@ -161,15 +167,13 @@ class Caddy {
     })
 
     // wait until running
-    await new Promise((resolve, reject) => {
-      ondata({ raw: "waiting until caddy server is up...\r\n" })
-      setInterval(async () => {
-        let running = await this.running()
-        if (running) {
-          resolve()
-        }
-      }, 2000)
-    })
+    ondata({ raw: "waiting until caddy server is up...\r\n" })
+    const running = await this.waitForAdmin()
+    if (!running) {
+      const message = "caddy admin did not become available. Certificate trust may be waiting for system approval."
+      ondata({ raw: `${message}\r\n` })
+      throw new Error(message)
+    }
     ondata({ raw: "caddy is running!\r\n" })
 
     if (this.kernel.platform === "win32") {

package/kernel/bin/huggingface.js

@@ -3,7 +3,7 @@ class Huggingface {
   description = "Installs huggingface_hub for downloading models and assets from Hugging Face."
   cmd() {
     //return 'huggingface_hub "hf-xet!=1.1.10"'
-    return 'huggingface_hub=1.0.1'
+    return 'huggingface_hub=1.20.1'
   }
   async install(req, ondata) {
     await this.kernel.bin.exec({
@@ -18,7 +18,7 @@ class Huggingface {
       let version = this.kernel.bin.installed.conda_versions.huggingface_hub
       if (version) {
         let coerced = semver.coerce(version)
-        if (coerced && semver.eq(coerced, "1.0.1")) {
+        if (coerced && semver.eq(coerced, "1.20.1")) {
           return true
         }
       }

package/kernel/bin/setup.js

@@ -280,7 +280,7 @@ module.exports = {
       { name: "git", },
       { name: "uv", },
       { name: "node", },
-      { name: "caddy", },
+      { name: "huggingface" },
       { name: "py", },
       { name: "bluefairy" },
     ])
@@ -293,8 +293,7 @@ module.exports = {
         zip_cmd,
         "huggingface",
         "uv",
-        "git",
-        "caddy"
+        "git"
       ]
     }
   },

package/kernel/bin/uv.js

@@ -1,11 +1,14 @@
-const fs = require('fs')
-const fetch = require('cross-fetch')
-const { rimraf } = require('rimraf')
-const decompress = require('decompress');
+const UV_VERSION = "0.11.23"
+
+const normalizeVersion = (version) => {
+  const match = String(version || "").match(/^\d+\.\d+\.\d+/)
+  return match ? match[0] : null
+}
+
 class UV {
   description = "Installs uv, a fast Python package and virtual environment manager."
   cmd() {
-    return "uv"
+    return `uv=${UV_VERSION}`
   }
   async install(req, ondata) {
     await this.kernel.bin.exec({
@@ -16,7 +19,11 @@ class UV {
     }, ondata)
   }
   async installed() {
-    return this.kernel.bin.installed.conda.has("uv")
+    if (!this.kernel.bin.installed.conda.has("uv")) {
+      return false
+    }
+    let version = this.kernel.bin.installed.conda_versions && this.kernel.bin.installed.conda_versions.uv
+    return normalizeVersion(version) === UV_VERSION
   }
   async uninstall(req, ondata) {
     await this.kernel.bin.exec({

package/kernel/connect/index.js

@@ -9,7 +9,11 @@ class Connect {
     this.config = config
     this.clients = {}
     for(let name in this.config) {
-      this.clients[name] = new Backend(kernel, name, this.config[name])
+      if (name === "huggingface") {
+        this.clients[name] = new Huggingface(kernel, this.config[name])
+      } else {
+        this.clients[name] = new Backend(kernel, name, this.config[name])
+      }
     }
   }
   async profile(provider, req) {

package/kernel/connect/providers/huggingface/index.js

@@ -1,97 +1,235 @@
-const fetch = require('cross-fetch')
+const { execFile, spawn } = require('child_process')
 const fs = require('fs')
 const path = require('path')
+const fetch = require('cross-fetch')
+const Environment = require('../../../environment')
+
+const stripAnsi = (value) => String(value || "").replace(/\x1B\[[0-?]*[ -/]*[@-~]/g, "")
+
 class Huggingface {
-  CLIENT_ID = 'e90d4a4d-68a6-4c12-ae71-64756b5918de'
-  REDIRECT_URI = 'https://pinokio.localhost/connect/huggingface'
-  HF_OAUTH_URL = 'https://huggingface.co/oauth/authorize'
-  HF_TOKEN_URL = 'https://huggingface.co/oauth/token'
-  HF_API_URL = 'https://huggingface.co/api/whoami-v2'
-  constructor(kernel) {
+  constructor(kernel, config) {
     this.kernel = kernel
+    this.config = config
+    this.loginSession = null
   }
   async readme() {
     return ""
   }
-  async persist(auth) {
-    console.log("PERSIST", auth)
-    this.auth = auth
-    this.auth.expires_at = Date.now() + (this.auth.expires_in * 1000);
-    let authPath = this.kernel.path('connect/huggingface.json')
-    await fs.promises.mkdir(this.kernel.path("connect"), { recursive: true }).catch((e) => { })
-    await fs.promises.writeFile(authPath, JSON.stringify(this.auth, null, 2))
-
-    // huggingface-cli login
-    await this.kernel.exec({
-      message: `hf auth login --token ${this.auth.access_token} --add-to-git-credential`
-    }, (stream) => {
-      process.stdout.write(stream.raw)
+  defaultTokenPath() {
+    return this.kernel.path("cache", "HF_AUTH", "token")
+  }
+  async authEnv() {
+    let systemEnv = {}
+    try {
+      systemEnv = await Environment.get(this.kernel.homedir, this.kernel)
+    } catch (_) {
+      systemEnv = {}
+    }
+    const env = Object.assign({}, process.env, systemEnv)
+    if (!env.HF_TOKEN_PATH) {
+      env.HF_TOKEN_PATH = this.defaultTokenPath()
+    }
+    if (!env.HF_HUB_DISABLE_UPDATE_CHECK) {
+      env.HF_HUB_DISABLE_UPDATE_CHECK = "1"
+    }
+    delete env.HF_TOKEN
+    delete env.HUGGING_FACE_HUB_TOKEN
+    await fs.promises.mkdir(path.dirname(env.HF_TOKEN_PATH), { recursive: true }).catch(() => {})
+    return env
+  }
+  hfPath() {
+    const candidates = this.kernel.platform === "win32" ? [
+      this.kernel.path("bin", "miniconda", "Scripts", "hf.exe"),
+      this.kernel.path("bin", "miniconda", "Scripts", "hf"),
+      this.kernel.path("bin", "miniconda", "bin", "hf"),
+    ] : [
+      this.kernel.path("bin", "miniconda", "bin", "hf"),
+    ]
+    for (const candidate of candidates) {
+      if (fs.existsSync(candidate)) {
+        return candidate
+      }
+    }
+    return candidates[0]
+  }
+  async runHf(args) {
+    const env = await this.authEnv()
+    return new Promise((resolve, reject) => {
+      execFile(this.hfPath(), args, {
+        cwd: this.kernel.homedir,
+        env,
+        maxBuffer: 1024 * 1024,
+      }, (error, stdout, stderr) => {
+        if (error) {
+          error.stdout = stdout
+          error.stderr = stderr
+          reject(error)
+        } else {
+          resolve({ stdout, stderr, env })
+        }
+      })
     })
   }
-  async destroy() {
-    await fs.promises.rm(this.kernel.path("connect/huggingface.json"))
-    this.auth = null
+  serializeLoginSession(session) {
+    if (!session) {
+      return null
+    }
+    return {
+      status: session.status,
+      login: session.login,
+      error: session.error,
+      token_path: session.tokenPath,
+    }
   }
-  async sync() {
-    // check if auth exists
-    //  if not, throw error
-    let authPath = this.kernel.path('connect/huggingface.json')
-    this.auth = (await this.kernel.loader.load(authPath)).resolved
-    if (!this.auth) {
+  parseDeviceLogin(output) {
+    const clean = stripAnsi(output).replace(/\s+/g, " ")
+    for (const line of stripAnsi(output).split(/\r?\n/)) {
+      if (!line.trim()) {
+        continue
+      }
+      let event
+      try {
+        event = JSON.parse(line)
+      } catch (_) {
+        continue
+      }
+      if (event && event.event === "device_code" && event.user_code) {
+        return {
+          verification_uri_complete: event.verification_uri_complete || event.verification_uri,
+          user_code: event.user_code,
+          expires_in: event.expires_in || null,
+          interval: event.interval || null,
+        }
+      }
+    }
+    const urlMatch = clean.match(/open\s+(https?:\/\/\S+)/i)
+    const codeMatch = clean.match(/enter the code\s+([A-Z0-9-]+)/i)
+    const expiresMatch = clean.match(/expires in\s+(\d+)\s+seconds/i)
+    if (!urlMatch || !codeMatch) {
       return null
     }
-    if (!this.auth.refresh_token) {
-      console.log("no refresh token")
+    return {
+      verification_uri_complete: urlMatch[1].replace(/[).,]+$/, ""),
+      user_code: codeMatch[1],
+      expires_in: expiresMatch ? Number(expiresMatch[1]) : null,
+    }
+  }
+  async login() {
+    if (this.loginSession && this.loginSession.status === "pending") {
+      return this.serializeLoginSession(this.loginSession)
+    }
+    const env = await this.authEnv()
+    const session = {
+      status: "pending",
+      login: null,
+      error: null,
+      output: "",
+      tokenPath: env.HF_TOKEN_PATH,
+      child: null,
+      readyResolved: false,
+    }
+    this.loginSession = session
+    let resolveReady
+    const ready = new Promise((resolve) => {
+      resolveReady = resolve
+    })
+    const markReady = () => {
+      if (!session.readyResolved) {
+        session.readyResolved = true
+        resolveReady()
+      }
+    }
+    const appendOutput = (chunk) => {
+      session.output += stripAnsi(chunk)
+      const parsed = this.parseDeviceLogin(session.output)
+      if (parsed && !session.login) {
+        session.login = parsed
+        markReady()
+      }
+    }
+    const child = spawn(this.hfPath(), ["auth", "login", "--format", "agent", "--force"], {
+      cwd: this.kernel.homedir,
+      env,
+      windowsHide: false,
+    })
+    session.child = child
+    child.stdout.on("data", appendOutput)
+    child.stderr.on("data", appendOutput)
+    child.on("error", (error) => {
+      session.status = "error"
+      session.error = error.message
+      markReady()
+    })
+    const readyTimer = setTimeout(() => {
+      if (session.status === "pending" && !session.login) {
+        session.status = "error"
+        session.error = "Timed out waiting for Hugging Face login code."
+        child.kill()
+        markReady()
+      }
+    }, 15000)
+    child.on("close", (code) => {
+      clearTimeout(readyTimer)
+      if (code === 0) {
+        session.status = "success"
+      } else {
+        session.status = "error"
+        session.error = session.error || stripAnsi(session.output).trim() || `hf auth login exited with code ${code}`
+      }
+      markReady()
+    })
+    await ready
+    if (session.status === "error") {
+      return this.serializeLoginSession(session)
+    }
+    return this.serializeLoginSession(session)
+  }
+  async keys() {
+    if (this.loginSession && this.loginSession.status === "pending") {
+      return this.serializeLoginSession(this.loginSession)
+    }
+    try {
+      const { stdout, env } = await this.runHf(["auth", "token", "--format", "quiet"])
+      const access_token = stripAnsi(stdout).trim().split(/\r?\n/).find(Boolean)
+      if (!access_token) {
+        return null
+      }
+      return { access_token, token_path: env.HF_TOKEN_PATH }
+    } catch (error) {
       return null
     }
-
-    // check if auth has expired
-    //  if expired, refresh and return
-    if (Date.now() < this.auth.expires_at) {
-      return
-    }
-    console.log("auth expired. refresh....", JSON.stringify({ auth: this.auth, id: this.CLIENT_ID }, null, 2))
-    // expired — refresh
-    const response = await fetch(this.HF_TOKEN_URL, {
-      method: 'POST',
-      headers: { 'Content-Type': 'application/x-www-form-urlencoded', },
-      body: new URLSearchParams({
-        grant_type: 'refresh_token',
-        refresh_token: this.auth.refresh_token,
-        client_id: this.CLIENT_ID,
-      })
+  }
+  async profile() {
+    const keys = await this.keys()
+    if (!keys || !keys.access_token) {
+      return null
+    }
+    const response = await fetch(this.config.profile.url, {
+      headers: {
+        'Authorization': 'Bearer ' + keys.access_token
+      }
     }).then((res) => {
+      if (!res.ok) {
+        throw new Error(`Hugging Face profile request failed with ${res.status}`)
+      }
       return res.json()
-    });
-    await this.persist(response)
-    return this.auth
-  }
-  async keys() {
-    await this.sync()
-    return this.auth
+    })
+    const connectPath = this.kernel.path("connect", "huggingface")
+    await fs.promises.mkdir(connectPath, { recursive: true }).catch(() => {})
+    await this.config.profile.cache(response, connectPath).catch(() => {})
+    return this.config.profile.render(response)
   }
-  async login(req) {
-    console.log("huggingface login", req)
-    await this.persist(req)
-    return this.auth
+  async destroy() {
+    if (this.loginSession && this.loginSession.child && this.loginSession.status === "pending") {
+      this.loginSession.child.kill()
+    }
+    this.loginSession = null
+    await this.runHf(["auth", "logout"]).catch(() => {})
+    await fs.promises.rm(this.kernel.path("connect", "huggingface"), { recursive: true, force: true }).catch(() => {})
   }
-//  async login (req) {
-//    const authHeader = 'Basic ' + Buffer.from(`${this.id}:`).toString('base64');
-//    const response = await fetch('https://api.x.com/2/oauth2/token', {
-//      method: 'POST',
-//      headers: {
-//        'Content-Type': 'application/json',
-////        'Authorization': authHeader
-//      },
-//      body: JSON.stringify(req.payload)
-//    }).then((res) => {
-//      return res.json()
-//    });
-//    await this.persist(response)
-//    return this.auth
-//  }
-  async logout (req) {
-    await this.destroy() 
+  async logout() {
+    await this.destroy()
   }
 }
+
 module.exports = Huggingface

package/kernel/environment.js

@@ -401,6 +401,21 @@ const ENVS = async () => {
       "#",
       "##########################################################################",
     ],
+  }, {
+    type: ["system"],
+    key: "HF_TOKEN_PATH",
+    val: "./cache/HF_AUTH/token",
+    comment: [
+      "##########################################################################",
+      "#",
+      "# HF_TOKEN_PATH",
+      "#",
+      "# Hugging Face authentication token file",
+      "# Pinokio keeps this shared across apps so each app can customize HF_HOME",
+      "# for model/cache files without changing the logged-in Hugging Face account.",
+      "#",
+      "##########################################################################",
+    ],
   }, {
     type: ["system", "app"],
     key: "TORCH_HOME",
@@ -535,7 +550,7 @@ const init_folders = async (homedir, kernel) => {
     if (is_absolute || is_relative) {
 
       // skip condarc and pipconfig => special case
-      if (["PIP_CONFIG_FILE", "CONDARC"].includes(key)) {
+      if (["PIP_CONFIG_FILE", "CONDARC", "HF_TOKEN_PATH"].includes(key)) {
         continue
       }
       // it's a path

package/kernel/router/localhost_home_router.js

@@ -20,6 +20,13 @@ class LocalhostHomeRouter {
             ]
           }
         },
+        "pki": {
+          "certificate_authorities": {
+            "local": {
+              "install_trust": false
+            }
+          }
+        },
 
         "http": {
           "servers": {

package/kernel/shell.js

@@ -208,11 +208,6 @@ class Shell {
     let system_env = await Environment.get(this.kernel.homedir, this.kernel)
     this.env = Object.assign(this.env, system_env)
 
-    let hf_keys = await this.kernel.connect.keys("huggingface")
-    if (hf_keys && hf_keys.access_token) {
-      this.env.HF_TOKEN = hf_keys.access_token
-    }
-
     const parentPath = (params.$parent && params.$parent.path) ? path.resolve(params.$parent.path) : null
     const apiRoot = path.resolve(this.kernel.path("api")) + path.sep
     const parentStat = parentPath ? await fs.promises.stat(parentPath).catch(() => null) : null
@@ -295,6 +290,7 @@ class Shell {
     }
 
     setDefaultEnvValue(this.env, "HF_HUB_DISABLE_UPDATE_CHECK", "1")
+    setDefaultEnvValue(this.env, "HF_TOKEN_PATH", path.resolve(this.kernel.homedir, "cache", "HF_AUTH", "token"))
 
     if (this.platform === "win32") {
       // Hugging Face file symlinks regularly fail on non-admin Windows setups.

package/kernel/util.js

@@ -1208,6 +1208,7 @@ const rewrite_localhost= (kernel, obj, source) => {
           }
         } else {
           let proxyPort = kernel.peer.info[kernel.peer.host].port_mapping["" + port]
+          if (!proxyPort) return url
           u.hostname = sourceIp;
           u.port = proxyPort;
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "7.3.10",
+  "version": "7.3.11",
   "description": "",
   "main": "index.js",
   "scripts": {