pinokiod 3.41.0 → 3.43.0

package/kernel/connect/backend.js

@@ -0,0 +1,110 @@
+const fetch = require('cross-fetch')
+const fs = require('fs')
+const path = require('path')
+class Backend {
+  constructor(kernel, name, config) {
+    this.kernel = kernel
+    this.name = name
+    this.config = config
+  }
+  async profile() {
+    let connectPath = this.kernel.path(`connect/${this.name}`)
+    let profilePath = path.resolve(connectPath, "profile.json")
+    this.profile_config = (await this.kernel.loader.load(profilePath)).resolved
+    if (!this.profile_config) {
+      await fs.promises.mkdir(connectPath, { recursive: true }).catch((e) => { })
+      let response = await fetch(this.config.profile.url, {
+        headers: {
+          'Authorization': 'Bearer ' + this.auth.access_token
+        }
+      }).then((res) => {
+        return res.json()
+      })
+      this.profile_config = response
+      await fs.promises.writeFile(profilePath, JSON.stringify(this.profile_config, null, 2))
+      let cwd = connectPath
+      await this.config.profile.cache(this.profile_config, cwd)
+    }
+    let rendered = this.config.profile.render(this.profile_config)
+    return rendered
+  }
+  async persist(auth) {
+    console.log("PERSIST", auth)
+    this.auth = auth
+    this.auth.expires_at = Date.now() + (this.auth.expires_in * 1000);
+    let connectPath = this.kernel.path(`connect/${this.name}`)
+    let authPath = path.resolve(connectPath, "auth.json")
+    await fs.promises.mkdir(connectPath, { recursive: true }).catch((e) => { })
+    await fs.promises.writeFile(authPath, JSON.stringify(this.auth, null, 2))
+  }
+  async destroy() {
+    await fs.promises.rm(this.kernel.path(`connect/${this.name}`), { recursive: true })
+    this.auth = null
+  }
+  async sync() {
+    // check if auth exists
+    //  if not, throw error
+    let authPath = this.kernel.path(`connect/${this.name}/auth.json`)
+    this.auth = (await this.kernel.loader.load(authPath)).resolved
+    if (!this.auth) {
+      console.log("not authenticated")
+      return null
+    }
+    if (!this.auth.refresh_token) {
+      console.log("no refresh token")
+      return null
+    }
+    if (!this.auth.access_token) {
+      console.log("no access token")
+      return null
+    }
+
+    // check if auth has expired
+    //  if expired, refresh and return
+    if (Date.now() < this.auth.expires_at) {
+      return
+    }
+    console.log("auth expired. refresh....", JSON.stringify({ auth: this.auth, id: this.config.CLIENT_ID }, null, 2))
+    // expired — refresh
+    const response = await fetch(this.config.TOKEN_URL, {
+      method: 'POST',
+      headers: {
+        'Content-Type': this.config.CONTENT_TYPE,
+        'Accept': "application/json"
+      },
+      body: new URLSearchParams({
+        grant_type: 'refresh_token',
+        refresh_token: this.auth.refresh_token,
+        client_id: this.config.CLIENT_ID,
+      })
+    }).then((res) => {
+      return res.json()
+    });
+    await this.persist(response)
+    return this.auth
+  }
+  async keys() {
+    await this.sync()
+    return this.auth
+  }
+  async login(req) {
+    console.log("login", this.name, req, this.config.TOKEN_URL, this.config.TOKEN_URL)
+    const response = await fetch(this.config.TOKEN_URL, {
+      method: 'POST',
+      headers: {
+        'Content-Type': this.config.CONTENT_TYPE,
+        'Accept': "application/json"
+      },
+      body: JSON.stringify(req)
+    }).then((res) => {
+      return res.json()
+    });
+    console.log("RESPONSE", response)
+    await this.persist(response)
+    return this.auth
+  }
+  async logout (req) {
+    await this.destroy() 
+  }
+}
+module.exports = Backend

package/kernel/connect/config.js

@@ -0,0 +1,171 @@
+const fs = require('fs')
+const path = require('path')
+const fetch = require('cross-fetch')
+module.exports = {
+  pinokio: {
+    CLIENT_ID: 'VmrH6TOG5Q68jTQKc9bP6hFl8oZ6LrkP',
+    REDIRECT_URI: 'https://pinokio.localhost/connect/pinokio',
+    OAUTH_URL: 'http://localhost:3001/oauth/authorize',
+    TOKEN_URL: 'http://localhost:3001/oauth/token',
+    CONTENT_TYPE: "application/json",
+    profile: {
+      url: 'http://localhost:3001/oauth/userinfo',
+      cache: async (response, cwd) => {
+        let url = response.avatar
+        let filename = url.split("/").pop()
+        const res = await fetch(url)
+        await fs.promises.writeFile(path.resolve(cwd, filename), Buffer.from(await res.arrayBuffer()))
+      },
+      render: (response) => {
+        console.log("RESPONSE", response)
+        let image = response.avatar.split("/").pop()
+        let imagePath = "/asset/connect/pinokio/" + image
+        return {
+          image: imagePath || '',
+          items: [{
+            key: "Username",
+            val: response.username || "N/A",
+          }, {
+            key: "Email",
+            val: response.email || "N/A"
+          }]
+        }
+      }
+    },
+    SCOPE: 'openid profile email read-repos write-repos manage-repos write-discussions read-billing inference-api jobs webhooks',
+  },
+  huggingface: {
+    CLIENT_ID: 'e90d4a4d-68a6-4c12-ae71-64756b5918de',
+    REDIRECT_URI: 'https://pinokio.localhost/connect/huggingface',
+    OAUTH_URL: 'https://huggingface.co/oauth/authorize',
+    TOKEN_URL: 'https://huggingface.co/oauth/token',
+    CONTENT_TYPE: "application/json",
+    profile: {
+      url: 'https://huggingface.co/api/whoami-v2',
+      cache: async (response, cwd) => {
+        let url = response.avatarUrl
+        let filename = url.split("/").pop()
+        const res = await fetch(url)
+        await fs.promises.writeFile(path.resolve(cwd, filename), Buffer.from(await res.arrayBuffer()))
+      },
+      render: (response) => {
+        let image = response.avatarUrl.split("/").pop()
+        let imagePath = "/asset/connect/huggingface/" + image
+        return {
+          image: imagePath || '',
+          items: [{
+            key: "Username",
+            val: response.name || "N/A",
+          }, {
+            key: "Full name",
+            val: response.fullname || "N/A"
+          }, {
+            key: "Email",
+            val: response.email || "N/A"
+          }]
+        }
+      }
+    },
+    SCOPE: 'openid profile email read-repos write-repos manage-repos write-discussions read-billing inference-api jobs webhooks',
+  },
+  x: {
+    CLIENT_ID: 'd2FQZ0U4NXpzYnRyS1hZeHBvbUc6MTpjaQ',
+    REDIRECT_URI: 'https://pinokio.localhost/connect/x',
+    OAUTH_URL: 'https://x.com/i/oauth2/authorize',
+    TOKEN_URL: 'https://api.twitter.com/2/oauth2/token',
+    //CONTENT_TYPE: "application/x-www-form-urlencoded",
+    CONTENT_TYPE: "application/json",
+    SCOPE: 'tweet.write tweet.read users.read bookmark.write bookmark.read like.write like.read media.write offline.access',
+    profile: {
+      url: 'https://api.twitter.com/2/users/me?user.fields=profile_image_url,username',
+      cache: async (response, cwd) => {
+        let url = response.data.profile_image_url
+        let filename = url.split("/").pop()
+        console.log("Fetching", { url, filename })
+        const res = await fetch(url)
+        await fs.promises.writeFile(path.resolve(cwd, filename), Buffer.from(await res.arrayBuffer()))
+      },
+      render: (response) => {
+        let image = response.data.profile_image_url.split("/").pop()
+        let imagePath = "/asset/connect/x/" + image
+        return {
+          image: imagePath || "",
+          items: [{
+            key: "Username",
+            val: response.data.username || "N/A"
+          }]
+        }
+      }
+    }
+  },
+  google: {
+    CLIENT_ID: '911627394513-75l6eumucknc8750pn5r5cog5sclndkr.apps.googleusercontent.com',
+    REDIRECT_URI: 'https://pinokio.localhost/connect/google',
+    OAUTH_URL: 'https://accounts.google.com/o/oauth2/v2/auth',
+    TOKEN_URL: 'https://oauth2.googleapis.com/token',
+    CONTENT_TYPE: "application/json",
+    SCOPE: 'openid profile email https://www.googleapis.com/auth/youtube https://www.googleapis.com/auth/youtube.upload https://www.googleapis.com/auth/youtube.force-ssl https://www.googleapis.com/auth/drive https://www.googleapis.com/auth/drive.file https://www.googleapis.com/auth/blogger https://www.googleapis.com/auth/photoslibrary https://www.googleapis.com/auth/photoslibrary.sharing https://www.googleapis.com/auth/calendar https://www.googleapis.com/auth/gmail.compose https://www.googleapis.com/auth/gmail.send',
+    profile: {
+      url: 'https://www.googleapis.com/oauth2/v2/userinfo',
+      render: (response) => {
+        return `<p><strong>Name:</strong> ${response.name || 'N/A'}</p>
+<p><strong>Email:</strong> ${response.email || 'N/A'}</p>
+<p><strong>Avatar:</strong> <img src="${response.picture || ''}" alt="Avatar" style="width: 40px; height: 40px; border-radius: 50%; vertical-align: middle;"></p>`
+      }
+    }
+  },
+  github: {
+    CLIENT_ID: 'Ov23cthkE6o0xkxngT2r',
+    REDIRECT_URI: 'https://pinokio.localhost/connect/github',
+    OAUTH_URL: 'https://github.com/login/oauth/authorize',
+    TOKEN_URL: 'https://github.com/login/oauth/access_token',
+    SCOPE: 'user:email read:user repo delete_repo admin:org admin:public_key admin:repo_hook admin:org_hook gist notifications workflow write:packages read:packages write:discussion read:discussion project admin:gpg_key codespace',
+    //CONTENT_TYPE: "application/x-www-form-urlencoded",
+    CONTENT_TYPE: "application/json",
+    profile: {
+      url: 'https://api.github.com/user',
+      render: (response) => {
+        return `<p><strong>Username:</strong> ${response.login || 'N/A'}</p>
+<p><strong>Name:</strong> ${response.name || 'N/A'}</p>
+<p><strong>Email:</strong> ${response.email || 'N/A'}</p>
+<p><strong>Avatar:</strong> <img src="${response.avatar_url || ''}" alt="Avatar" style="width: 40px; height: 40px; border-radius: 50%; vertical-align: middle;"></p>`
+      }
+    }
+  },
+  spotify: {
+    CLIENT_ID: '',
+    REDIRECT_URI: 'https://pinokio.localhost/connect/spotify',
+    OAUTH_URL: 'https://accounts.spotify.com/authorize',
+    TOKEN_URL: 'https://accounts.spotify.com/api/token',
+    CONTENT_TYPE: "application/x-www-form-urlencoded",
+    SCOPE: 'user-read-private user-read-email playlist-read-private playlist-read-collaborative playlist-modify-private playlist-modify-public user-library-modify user-library-read user-follow-modify user-follow-read',
+    profile: {
+      url: 'https://api.spotify.com/v1/me',
+      render: (response) => {
+        const avatarUrl = response.images && response.images.length > 0 
+          ? response.images[0].url 
+          : '';
+        return `<p><strong>Username:</strong> ${response.id || 'N/A'}</p>
+<p><strong>Display Name:</strong> ${response.display_name || 'N/A'}</p>
+<p><strong>Email:</strong> ${response.email || 'N/A'}</p>
+<p><strong>Avatar:</strong> <img src="${avatarUrl}" alt="Avatar" style="width: 40px; height: 40px; border-radius: 50%; vertical-align: middle;"></p>`
+      }
+    }
+  },
+  microsoft: {
+    CLIENT_ID: '',
+    REDIRECT_URI: 'https://pinokio.localhost/connect/microsoft',
+    OAUTH_URL: 'https://login.microsoftonline.com/common/oauth2/v2.0/authorize',
+    TOKEN_URL: 'https://login.microsoftonline.com/common/oauth2/v2.0/token',
+    SCOPE: 'openid profile email User.Read User.ReadWrite Files.ReadWrite Sites.ReadWrite.All Mail.ReadWrite Mail.Send Calendars.ReadWrite Tasks.ReadWrite Notes.ReadWrite.All OnlineMeetings.ReadWrite',
+    CONTENT_TYPE: "application/x-www-form-urlencoded",
+    profile: {
+      url: 'https://graph.microsoft.com/v1.0/me',
+      render: (response) => {
+        return `<p><strong>Name:</strong> ${response.displayName || 'N/A'}</p>
+<p><strong>Email:</strong> ${response.mail || response.userPrincipalName || 'N/A'}</p>
+<p><strong>Username:</strong> ${response.userPrincipalName || 'N/A'}</p>`
+      }
+    }
+  }
+}

package/kernel/connect/index.js

@@ -1,24 +1,35 @@
 const fetch = require('cross-fetch')
 const X = require('./providers/x')
+const Huggingface = require('./providers/huggingface')
+const config = require('./config')
+const Backend = require('./backend')
 class Connect {
   constructor(kernel) {
     this.kernel = kernel
-    this.x = new X(kernel)
+    this.config = config
+    this.clients = {}
+    for(let name in this.config) {
+      this.clients[name] = new Backend(kernel, name, this.config[name])
+    }
   }
-  async request(provider, method, req) {
-    let res = await this[provider].request(method, req)
-    return res
+  async profile(provider, req) {
+    if (this.clients[provider] && this.clients[provider].profile) {
+      let res = await this.clients[provider].profile()
+      return res
+    } else {
+      return null
+    }
   }
   async login(provider, req) {
-    let res = await this[provider].login(req)
+    let res = await this.clients[provider].login(req)
     return res
   }
   async logout(provider, req) {
-    let res = await this[provider].logout(req)
+    let res = await this.clients[provider].logout(req)
     return res
   }
   async keys(provider) {
-    let res = await this[provider].keys()
+    let res = await this.clients[provider].keys()
     return res
   }
 }

package/kernel/connect/providers/huggingface/index.js

@@ -0,0 +1,98 @@
+const fetch = require('cross-fetch')
+const fs = require('fs')
+const path = require('path')
+class Huggingface {
+  CLIENT_ID = 'e90d4a4d-68a6-4c12-ae71-64756b5918de'
+  REDIRECT_URI = 'https://pinokio.localhost/connect/huggingface'
+  HF_OAUTH_URL = 'https://huggingface.co/oauth/authorize'
+  HF_TOKEN_URL = 'https://huggingface.co/oauth/token'
+  HF_API_URL = 'https://huggingface.co/api/whoami-v2'
+  constructor(kernel) {
+    this.kernel = kernel
+  }
+  async readme() {
+    return ""
+  }
+  async persist(auth) {
+    console.log("PERSIST", auth)
+    this.auth = auth
+    this.auth.expires_at = Date.now() + (this.auth.expires_in * 1000);
+    let authPath = this.kernel.path('connect/huggingface.json')
+    await fs.promises.mkdir(this.kernel.path("connect"), { recursive: true }).catch((e) => { })
+    await fs.promises.writeFile(authPath, JSON.stringify(this.auth, null, 2))
+
+    // huggingface-cli login
+    await this.kernel.exec({
+      message: `hf auth login --token ${this.auth.access_token} --add-to-git-credential`
+    }, (stream) => {
+      process.stdout.write(stream.raw)
+    })
+  }
+  async destroy() {
+    await fs.promises.rm(this.kernel.path("connect/huggingface.json"))
+    this.auth = null
+  }
+  async sync() {
+    // check if auth exists
+    //  if not, throw error
+    let authPath = this.kernel.path('connect/huggingface.json')
+    this.auth = (await this.kernel.loader.load(authPath)).resolved
+    if (!this.auth) {
+      console.log("not authenticated")
+      return null
+    }
+    if (!this.auth.refresh_token) {
+      console.log("no refresh token")
+      return null
+    }
+
+    // check if auth has expired
+    //  if expired, refresh and return
+    if (Date.now() < this.auth.expires_at) {
+      return
+    }
+    console.log("auth expired. refresh....", JSON.stringify({ auth: this.auth, id: this.CLIENT_ID }, null, 2))
+    // expired — refresh
+    const response = await fetch(this.HF_TOKEN_URL, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/x-www-form-urlencoded', },
+      body: new URLSearchParams({
+        grant_type: 'refresh_token',
+        refresh_token: this.auth.refresh_token,
+        client_id: this.CLIENT_ID,
+      })
+    }).then((res) => {
+      return res.json()
+    });
+    await this.persist(response)
+    return this.auth
+  }
+  async keys() {
+    await this.sync()
+    return this.auth
+  }
+  async login(req) {
+    console.log("huggingface login", req)
+    await this.persist(req)
+    return this.auth
+  }
+//  async login (req) {
+//    const authHeader = 'Basic ' + Buffer.from(`${this.id}:`).toString('base64');
+//    const response = await fetch('https://api.x.com/2/oauth2/token', {
+//      method: 'POST',
+//      headers: {
+//        'Content-Type': 'application/json',
+////        'Authorization': authHeader
+//      },
+//      body: JSON.stringify(req.payload)
+//    }).then((res) => {
+//      return res.json()
+//    });
+//    await this.persist(response)
+//    return this.auth
+//  }
+  async logout (req) {
+    await this.destroy() 
+  }
+}
+module.exports = Huggingface

package/kernel/connect/providers/x/index.js

@@ -44,7 +44,6 @@ class X {
     // check if auth has expired
     //  if expired, refresh and return
     if (Date.now() < this.auth.expires_at) {
-      console.log("authentication valid")
       return
     }
 

package/kernel/environment.js

@@ -299,7 +299,7 @@ const ENVS = async () => {
 //}
 
 // type := system|app
-const ENV = async (type, homedir) => {
+const ENV = async (type, homedir, kernel) => {
   const envs = await ENVS()
   let filtered_envs = []
   let irrelevant_keys = []
@@ -334,11 +334,9 @@ const ENV = async (type, homedir) => {
     //  if e.key exists on system env, use that
     //  if e.key does NOT exist on system env, use from the hardcoded default option
     if (type === 'app') {
-      system_env = await get_raw(homedir)
+      system_env = await get_raw(homedir, kernel)
       if (e.key in system_env) {
-//        console.log(`original ${e.key}=${val}`)
         val = system_env[e.key]
-//        console.log(`inherited from system_env: ${e.key}=${val}`)
         keys.add(e.key)
       }
     }
@@ -353,14 +351,10 @@ const ENV = async (type, homedir) => {
       if (!keys.has(key)) {
         // the key has not been processed, need to add to the lines
         if (irrelevant_keys.includes(key)) {
-          // if the key was explicitly stated to be not included, skip
-//          console.log("irrelevant key", key)
         } else {
-//          console.log("relevant key", key)
           let val = system_env[key]
           let kv = `${key}=${val}`
           lines.push(kv)
-//          console.log(`inherited custom environment key from system_env: ${kv}`)
         }
       }
     }
@@ -368,8 +362,8 @@ const ENV = async (type, homedir) => {
 
   return lines.join("\n")
 }
-const init_folders = async (homedir) => {
-  const current_env = await get(homedir)
+const init_folders = async (homedir, kernel) => {
+  const current_env = await get(homedir, kernel)
   for(let key in current_env) {
     let val = current_env[key]
 
@@ -391,8 +385,8 @@ const init_folders = async (homedir) => {
 // Get the actual environment variable at specific path
 const get2 = async (filepath, kernel) => {
   let api_path = Util.api_path(filepath, kernel)
-  let default_env = await get(kernel.homedir)
-  let api_env = await get(api_path)
+  let default_env = await get(kernel.homedir, kernel)
+  let api_env = await get(api_path, kernel)
   let process_env = kernel.envs || process.env
   let current_env = Object.assign({}, process_env, default_env, api_env)
   for(let key in current_env) {
@@ -407,8 +401,10 @@ const get2 = async (filepath, kernel) => {
 // return env object
 // 1. if the value starts with ./ => convert to absolute path
 // 2. if the value is empty => don't return the kv pair for that value
-const get = async (homedir) => {
-  const env_path = path.resolve(homedir, "ENVIRONMENT")
+const get = async (homedir, kernel) => {
+  const got_root = await get_root({ path: homedir }, kernel)
+  const root = got_root.root
+  const env_path = path.resolve(root, "ENVIRONMENT")
   const current_env = await Util.parse_env(env_path)
   for(let key in current_env) {
     let val = current_env[key]
@@ -423,8 +419,10 @@ const get = async (homedir) => {
   return current_env
 }
 
-const get_raw = async (homedir) => {
-  const env_path = path.resolve(homedir, "ENVIRONMENT")
+const get_raw = async (homedir, kernel) => {
+  const got_root = await get_root({ path: homedir }, kernel)
+  const root = got_root.root
+  const env_path = path.resolve(root, "ENVIRONMENT")
   const current_env = await Util.parse_env(env_path)
   for(let key in current_env) {
     let val = current_env[key]
@@ -468,8 +466,6 @@ const requirements = async (script, cwd, kernel) => {
           } else {
             item.host = ""
           }
-          console.log({ env, env_key })
-
           if (env[env_key]) {
             item.val = env[env_key]
           } else {
@@ -485,4 +481,80 @@ const requirements = async (script, cwd, kernel) => {
   }
   return { items: pre_items, requires_instantiation }
 }
-module.exports = { ENV, get, get2, init_folders, requirements }
+const get_root = async (options, kernel) => {
+  let root
+  let relpath
+  if (options.path) {
+    let primary_path = path.resolve(options.path, "pinokio")
+    let primary_exists = await kernel.exists(primary_path)
+    if (primary_exists) {
+      root = primary_path
+      relpath = "pinokio"
+    } else {
+      root = options.path
+      relpath = ""
+    }
+  } else if (options.name) {
+    let primary_path = kernel.path("api", options.name, "pinokio")
+    let primary_exists = await kernel.exists(primary_path)
+    if (primary_exists) {
+      root = primary_path
+      relpath = "pinokio"
+    } else {
+      root = kernel.path("api", options.name)
+      relpath = ""
+    }
+  }
+  return { root, relpath }
+}
+const init = async (options, kernel) => {
+  /*
+  options = {
+    name,
+    no_inherit
+  }
+  */
+  // check if pinokio folder exists
+  // 1. if it exists, it's pinokio/ENVIRONMENT
+  // 2. if not, it's ENVIRONMENT
+  let relpath, root
+  if (options.name) {
+    let got_root = await get_root(options, kernel)
+    relpath = got_root.relpath
+    root = got_root.root
+  } else {
+    root = kernel.homedir
+  }
+  let current = path.resolve(root, "ENVIRONMENT")
+  let exists = await kernel.exists(current)
+  if (exists) {
+    // if ENVIRONMENT already exists, don't do anything
+  } else {
+    // if ENVIRONMENT doesn't exist, need to create one
+    // 1. if _ENVIRONMENT exists, create ENVIRONMENT by appending _ENVIRONMENT to ENVIRONMENT
+    // 2. if _ENVIRONMENT doesn't exist, just write ENVIRONMENT
+    // if _ENVIRONMENT exists, 
+    let _environment = path.resolve(root, "_ENVIRONMENT")
+    let _exists = await kernel.exists(_environment)
+    if (options && options.no_inherit) {
+      if (_exists) {
+        let _environmentStr = await fs.promises.readFile(_environment, "utf8")
+        await fs.promises.writeFile(current, _environmentStr)
+      }
+    } else {
+      let content = await ENV("app", kernel.homedir, kernel)
+      if (_exists) {
+        let _environmentStr = await fs.promises.readFile(_environment, "utf8")
+        await fs.promises.writeFile(current, _environmentStr + "\n\n\n" + content)
+      } else {
+        await fs.promises.writeFile(current, content)
+      }
+    }
+  }
+  return {
+    relpath,
+    root_path: root,
+    env_path: current
+  }
+}
+module.exports = { ENV, get, get2, init_folders, requirements, init, get_root  }