pinokiod 3.41.0 → 3.43.0

package/kernel/git.js

@@ -4,16 +4,18 @@ const path = require('path')
 const { glob, sync, hasMagic } = require('glob-gitignore')
 const http = require('isomorphic-git/http/node')
 const ini = require('ini')
+const Util = require('./util')
 class Git {
   constructor(kernel) {
     this.kernel = kernel
     this.dirs = new Set()
+    this.mapping = {}
   }
   async findGitDirs(dir, results = []) {
     const entries = await fs.promises.readdir(dir, { withFileTypes: true });
-
     for (const entry of entries) {
-      if (entry.isDirectory()) {
+      let type = await Util.file_type(dir, entry)
+      if (type.directory) {
         if (entry.name === '.git') {
           results.push(path.join(dir, entry.name));
           continue; // don't go deeper in this repo
@@ -26,6 +28,16 @@ class Git {
     }
     return results;
   }
+  async index(kernel) {
+    await this.repos(kernel.path("api"))
+  }
+  find(git_url) {
+    let found = this.mapping[git_url]
+    if (!found) {
+      found = this.mapping[git_url + ".git"]
+    }
+    return found
+  }
 
   async repos (root) {
     let _repos = await this.findGitDirs(root)
@@ -47,10 +59,13 @@ class Git {
       const gitParentRelPath = path.relative(this.kernel.path("api"), gitParentPath)
       let dir = path.dirname(gitPath)
       let display_name
+      let main
       if (gitRelPathSansGit === ".") {
         display_name = name
+        main = true
       } else {
         display_name = `${name}/${gitRelPathSansGit}`
+        main = false
       }
       try {
         let gitRemote = await git.getConfig({
@@ -60,16 +75,26 @@ class Git {
           path: 'remote.origin.url'
         })
         repos.push({
+          main,
           name: display_name,
           gitPath,
           gitRelPath,
           gitParentPath,
           gitParentRelPath,
           dir,
-          url: gitRemote
+          url: gitRemote,
         })
+        if (!this.mapping[gitRemote]) {
+          let head = await this.getHead(gitParentPath)
+          this.mapping[gitRemote] = {
+            main,
+            path: gitParentPath,
+            head 
+          }
+        }
       } catch (e) {
         repos.push({
+          main,
           name: display_name,
           gitPath,
           gitRelPath,
@@ -92,6 +117,24 @@ class Git {
       return null
     }
   }
+  async getHead(repoPath) {
+    const commits = await git.log({
+      fs,
+      dir: repoPath,
+      depth: 1,   // only get the latest commit
+    });
+
+    if (commits.length === 0) {
+      throw new Error("No commits found in repository");
+    }
+
+    const { oid, commit } = commits[0];
+    return {
+      hash: oid,
+      message: commit.message,
+    };
+  }
+
   async resolveCommitOid(dir, ref) {
     const oid = await git.resolveRef({ fs, dir, ref });
     const { type } = await git.readObject({ fs, dir, oid });

package/kernel/index.js

@@ -43,6 +43,7 @@ const { ProxyAgent } = require('proxy-agent');
 const fakeUa = require('fake-useragent');
 //const kill = require('./tree-kill');
 const kill = require('kill-sync')
+const ejs = require('ejs');
 const VARS = {
   pip: {
     install: {
@@ -83,11 +84,101 @@ class Kernel {
     this.jsdom = jsdom
     this.exposed = {}
     this.envs = {}
+    this.pinokio_configs = {}
     this.shellpath = shellPath.sync()
     this.favicon = new Favicon()
 
 
   }
+  async renderFile(filepath, data) {
+    let response = await new Promise((resolve, reject) => {
+      ejs.renderFile(filepath, data, (err, rendered) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(rendered)
+        }
+      })
+    })
+    return response
+  }
+  async dns(request) {
+    let config
+    let api_path
+    let name
+
+    let is_static_project   // is_static_project := no pinokio.js file
+    if (request.path) {
+      let relpath = path.relative(this.path("api"), request.path)
+      // chunks: "comfy.git/start.js"
+      let chunks = relpath.split(path.sep)
+      // name: "comfy.git"
+      name = chunks[0]
+      let launcher = await this.api.launcher(name)
+      config = launcher.script
+      api_path = launcher.root
+    } else {
+      config = request.config
+      name = request.name
+      api_path = this.path("api", request.name)
+    }
+    if (!config) {
+      is_static_project = true
+    }
+    let dns = {
+      "@": [
+        "$local.url@start",   // load local.url for "start" script 
+        ".",                  // load ./index.html
+        "dist",               // load ./dist/index.html
+        "build",              // load ./build/index.html
+        "docs"                // load ./docs/index.html
+      ]
+    }
+    if (config) {
+      if (config.dns) {
+        for(let key in config.dns) {
+          if (config.dns[key]) {
+            config.dns[key] = config.dns[key].concat(dns[key])
+          }
+        }
+      } else {
+        config.dns = dns
+      }
+    } else {
+      config = {
+        dns
+      }
+    }
+
+    for(let key in config.dns) {
+      let filtered = []
+      for(let item of config.dns[key]) {
+        if (item.startsWith("$")) {
+          if (is_static_project) {
+            // do not add since there will be no local variables
+          } else {
+            filtered.push(item)
+          }
+        } else if (item.startsWith(":")) {
+          // port
+          filtered.push(item)
+        } else {
+          // file path => check if the <path>/index.html exists
+          let exists = await this.exists(path.resolve(api_path, item, "index.html"))
+          if (exists) {
+            filtered.push(item)
+          }
+        }
+      }
+      if (!filtered.includes(".")) {
+        filtered.push(".")
+      }
+      config.dns[key] = filtered
+      
+    }
+    this.pinokio_configs[name] = config
+  }
+
   /*
     kernel.env() => return the system environment
     kernel.env("api/stablediffusion") => returns the full environment object
@@ -218,6 +309,9 @@ class Kernel {
       return 42003
     }
   }
+  async symlink({ from, to }) {
+    await Util.symlink({ from, to })
+  }
   port(port) {
     // 1. if port is passed, check port
     // 2. if no port is passed, get the next available port
@@ -292,7 +386,6 @@ class Kernel {
   }
   async refresh(notify_peers) {
     const ts = Date.now()
-//    console.time("Total " + ts)
     let network_active = await this.network_active()
     if (!network_active) {
       return
@@ -306,43 +399,33 @@ class Kernel {
       }
 
       // 1. get the process list
-//      console.time("> 1. Proc Refresh"+ts)
       await this.processes.refresh()
-//      console.timeEnd("> 1. Proc Refresh"+ts)
 
       // 2. refresh peer info to reflect the proc info
-//      console.time("> 2. Peer Refresh"+ts)
       //await this.peer.refresh()
       await this.peer.refresh_host(this.peer.host)
-//      console.timeEnd("> 2. Peer Refresh"+ts)
 
       // 3. load custom routers from ~/pinokio/network
-//      console.time("> 3. Router Init"+ts)
       await this.router.init()
-//      console.timeEnd("> 3. Router Init"+ts)
 
       // 4. set current local host router info
-//      console.time("> 4. Router Local"+ts)
       await this.router.local()
-//      console.timeEnd("> 4. Router Local"+ts)
 
       // 7. update remote router
-//      console.time("> 7. Router Remote"+ts)
       await this.router.remote()
-//      console.timeEnd("> 7. Router Remote"+ts)
+
+      await this.router.static()
 
       await this.router.custom_domain()
 
+
       this.router.fallback()
 
       // 8. update caddy config
-//      console.time("> 8. Router Update"+ts)
       await this.router.update()
-//      console.timeEnd("> 8. Router Update"+ts)
 
       // 6. tell peers to refresh
       let changed
-//      console.time("> 9. notify if changed")
       let new_config = JSON.stringify(await this.peer.current_host())
       if (this.old_config !== new_config) {
         console.log("Proc config has changed. update router.")
@@ -355,11 +438,9 @@ class Kernel {
       if (changed) {
         await this.peer.notify_refresh()
       }
-//      console.timeEnd("> 9. notify if changed")
       // 9. announce self to the peer network
       this.peer.announce()
     }
-//    console.timeEnd("Total " + ts)
   }
   async clearLog(group) {
     let relativePath = path.relative(this.homedir, group)
@@ -382,7 +463,6 @@ class Kernel {
     }
   }
   async download(options, ondata) {
-    console.log("download", { options })
     const agent = new ProxyAgent();
     const userAgent = fakeUa()
     let url = options.uri
@@ -396,7 +476,6 @@ class Kernel {
     if (options.filename) {
       opts.fileName = options.filename
     }
-    console.log("Download", opts)
     if (process.env.HTTP_PROXY && process.env.HTTP_PROXY.length > 0) {
       opts.httpRequestOptions = { agent }
       opts.httpsRequestOptions = { agent }
@@ -538,23 +617,24 @@ class Kernel {
       }
     }
 
+
     let files = await fs.promises.readdir(this.api.userdir, { withFileTypes: true })
-    let folders = files.filter((f) => { return f.isDirectory() }).map((x) => { return x.name })
+//    let folders = files.filter((f) => { return f.isDirectory() }).map((x) => { return x.name })
+    let folders = []
+    for(let file of files) {
+//    for(let folder of folders) {
+      let file_path = this.path("api", file.name)
+      let type = await Util.file_type(this.path("api"), file)
+      if (type.directory) {
+        folders.push(file.name)
+        await this.dns({
+          path: file_path
+        })
+      }
+    }
+
     let meta = {}
     for(let folder of folders) {
-
-      let pinokiojson = this.path("api", folder, "pinokio.json")
-      let pinokiometajson = this.path("api", folder, "pinokio_meta.json")
-      let pinokiojs = this.path("api", folder, "pinokio.js")
-      let exists1 = await this.exists(pinokiojson)
-      let exists2 = await this.exists(pinokiometajson)
-      let exists3 = await this.exists(pinokiojs)
-//      if (!exists1 && !exists2 && !exists3) {
-//        await fs.promises.writeFile(pinokiojson, JSON.stringify({
-//          title: "No title",
-//          description: ""
-//        }))
-//      }
       let pinokio = await this.api.meta(folder)
       if (pinokio) {
         i.api.push({
@@ -598,7 +678,6 @@ class Kernel {
     }
   }
   which(name, pattern) {
-    console.log("Which", { name, pattern })
     if (this.platform === "win32") {
       try {
         const result = execSync(`where ${name}`, { env: this.envs, encoding: "utf-8" })
@@ -755,7 +834,7 @@ class Kernel {
         // if it doesn't exist, write to ~/pinokio/ENVIRONMENT
         let e = await this.exists(this.homedir, "ENVIRONMENT")
         if (!e) {
-          let str = await Environment.ENV("system", this.homedir)
+          let str = await Environment.ENV("system", this.homedir, this)
           await fs.promises.writeFile(path.resolve(this.homedir, "ENVIRONMENT"), str)
         }
 
@@ -768,7 +847,7 @@ class Kernel {
         await Util.update_env(fullpath, updated)
 
         // 2. mkdir all the folders if not already created
-        await Environment.init_folders(this.homedir)
+        await Environment.init_folders(this.homedir, this)
 
         // if key.json doesn't exist, create an empty json file
         let ee = await this.exists(this.homedir, "key.json")
@@ -776,8 +855,6 @@ class Kernel {
           await fs.promises.writeFile(path.resolve(this.homedir, "key.json"), JSON.stringify({}))
         }
 
-
-
 //        // 3. check if Caddyfile exists
 //        let e2 = await this.exists(this.homedir, "Caddyfile")
 //        if (!e2) {
@@ -793,6 +870,9 @@ class Kernel {
 //      let contents = await fs.promises.readdir(this.homedir)
       //await this.bin.init()
       let ts = Date.now()
+      this.git.index(this).then(() => {
+        //console.log(this.git.mapping)
+      })
       this.bin.init().then(() => {
         if (this.homedir) {
           this.shell.init().then(async () => {
@@ -832,7 +912,7 @@ class Kernel {
 
               this.sysinfo = info
 
-              await this.getInfo()
+              await this.getInfo(true)
 
               await fs.promises.mkdir(this.path("logs"), { recursive: true }).catch((e) => { })
               await fs.promises.writeFile(this.path("logs/system.json"), JSON.stringify(this.i, null, 2))
@@ -845,7 +925,7 @@ class Kernel {
                   if (this.i) {
                     for(let api of this.i.api) {
                       let env_path = path.resolve(this.api.userdir, api.path)
-                      let e = await Environment.get(env_path)
+                      let e = await Environment.get(env_path, this)
                       if (e.PINOKIO_SCRIPT_AUTOLAUNCH && e.PINOKIO_SCRIPT_AUTOLAUNCH.trim().length > 0) {
                         let autolaunch_path = path.resolve(env_path, e.PINOKIO_SCRIPT_AUTOLAUNCH)
                         let exists = await this.exists(autolaunch_path)

package/kernel/peer.js

@@ -39,7 +39,7 @@ class PeerDiscovery {
     }
   }
   async check(kernel) {
-    let env = await Environment.get(kernel.homedir)
+    let env = await Environment.get(kernel.homedir, kernel)
     //let peer_active = true
     let peer_active = false
     if (env && env.PINOKIO_NETWORK_ACTIVE && (env.PINOKIO_NETWORK_ACTIVE==="1" || env.PINOKIO_NETWORK_ACTIVE.toLowerCase()==="true")) {
@@ -65,7 +65,7 @@ class PeerDiscovery {
     }
   }
   async start(kernel) {
-    let env = await Environment.get(kernel.homedir)
+    let env = await Environment.get(kernel.homedir, kernel)
 
     // by default expose to the local network
     //this.active = true
@@ -211,7 +211,7 @@ class PeerDiscovery {
     if (icon) {
       http_icon = `http://${this.host}:42000${icon}`;
       //https_icon = `https://${appname}.${this.name}.localhost/${iconpath}?raw=true`
-      https_icon = `https://pinokio.${this.name}.localhost/raw/${appname}/${iconpath}`
+      https_icon = `https://pinokio.${this.name}.localhost/asset/api/${appname}/${iconpath}`
     } else {
       for(let protocol of ["https", "http"]) {
         if (protocol === "https") {
@@ -254,9 +254,28 @@ class PeerDiscovery {
           if (external_port) {
             external_ip = `${this.host}:${external_port}`
           }
+          let internal_router = []
+          // check both the 127.0.0.1 and 0.0.0.0 for local ip
+          let a = router["127.0.0.1:" + proc.port]
+          let b = router["0.0.0.0:" + proc.port]
+          let c = router["localhost:" + proc.port]
+          if (a) {
+            internal_router = internal_router.concat(a)
+          }
+          if (b) {
+            internal_router = internal_router.concat(b)
+          }
+          if (c) {
+            internal_router = internal_router.concat(c)
+          }
+//          let ip = 
+//          if (router[proc.ip]) {
+//          }proc.
+
+
           let info = {
             external_router: router[external_ip] || [],
-            internal_router: router[proc.ip] || [],
+            internal_router,
             external_ip,
             external_port: parseInt(external_port),
             internal_port: parseInt(internal_port),
@@ -302,20 +321,23 @@ class PeerDiscovery {
       let https_icon = null
       let http_href = null
       let https_href = null
+      let app_href = null
       if (meta && !meta.init_required) {
         if (meta.title) {
           if (meta.icon) {
             http_icon = `http://${this.host}:42000${meta.icon}`;
             //https_icon = `https://${folder}.${this.name}.localhost/${meta.iconpath}?raw=true`
-            https_icon = `https://pinokio.${this.name}.localhost/raw/${folder}/${meta.iconpath}`
+            https_icon = `https://pinokio.${this.name}.localhost/asset/api/${folder}/${meta.iconpath}`
           }
           //https_href = `https://${folder}.${this.name}.localhost`
           https_href = `https://pinokio.${this.name}.localhost/p/${folder}`
           http_href = `http://${this.host}:42000/p/${folder}`
+          app_href = `https://${folder}.${this.name}.localhost`
           installed.push({
             folder,
             http_icon,
             https_icon,
+            app_href,
             https_href,
             http_href,
             ...meta 
@@ -329,13 +351,26 @@ class PeerDiscovery {
     let d = Date.now()
     let router_info = await this.router_info()
     let installed = await this.installed()
+    let peers = Object.values(this.info).filter((info) => {
+      return info.host !== this.host
+    }).map((info) => {
+      return {
+        name: info.name,
+        host: info.host
+      }
+    })
     return {
+      version: this.kernel.version,
       home: this.kernel.homedir,
       arch: this.kernel.arch,
       platform: this.kernel.platform,
+      gpu: this.kernel.gpu,
+      gpus: this.kernel.gpus,
       name: this.name,
       host: this.host,
+      peers: peers,
       port_mapping: this.kernel.router.port_mapping,
+      rewrite_mapping: this.kernel.router.rewrite_mapping,
       proc: this.kernel.processes.info,
       router: this.kernel.router.published(),
       router_info,

package/kernel/plugin.js

@@ -14,10 +14,11 @@ class Plugin {
     let plugins = []
     for(let plugin_path of plugin_paths) {
       let config = await this.kernel.require(path.resolve(plugin_dir, plugin_path))
-      let cwd = plugin_path.split("/").slice(0, -1).join("/")
+      let chunks = plugin_path.split(path.sep)
+      let cwd = chunks.slice(0, -1).join("/")
       config.image = "/asset/plugin/" + cwd + "/" + config.icon 
       plugins.push({
-        href: "/run/plugin/" + plugin_path,
+        href: "/run/plugin/" + chunks.join("/"),
         ...config
       })
     }

package/kernel/procs.js

@@ -13,7 +13,7 @@ class Procs {
     this.kernel = kernel
     this.cache = {}
   }
-  async isHttp(port) {
+  async isHttp(host, port) {
     // ignore caddy
     if (parseInt(port) === 2019) {
       return false
@@ -21,16 +21,16 @@ class Procs {
     if (this.cache.hasOwnProperty("" + port)) {
       return this.cache["" + port]
     }
-    let ip = "127.0.0.1"
+//    let ip = "127.0.0.1"
     let timeout = 1000
     let response = await new Promise(resolve => {
       const socket = new net.Socket();
       let response = '';
       let resolved = false;
       socket.setTimeout(timeout);
-      socket.connect(port, ip, () => {
+      socket.connect(port, host, () => {
         // Use a nonsense method to trigger a 400/405/501 from real HTTP servers
-        socket.write(`FOO / HTTP/1.1\r\nHost: ${ip}\r\nConnection: close\r\n\r\n`);
+        socket.write(`FOO / HTTP/1.1\r\nHost: ${host}\r\nConnection: close\r\n\r\n`);
       });
       socket.on('data', chunk => {
         response += chunk.toString();
@@ -73,7 +73,6 @@ class Procs {
     let pids = new Set()
     let s = stdout.trim()
     const lines = s.split('\n');
-//    console.time("###### Line parsing")
     for(let line of lines) {
       if (isWin) {
         // Skip headers
@@ -90,11 +89,17 @@ class Procs {
           }
 
           //if (state !== 'LISTENING') continue;
-          const chunks = localAddress.split(":")
-          const port = chunks.pop()
-          let ip = chunks.pop()
-          if (!ip || ip === "*") {
+          const chunks = /(.+):([0-9]+)/.exec(localAddress)
+          let host = chunks[1]
+          let port = chunks[2]
+          let ip
+
+          if (host === "*") {
             ip = "127.0.0.1:" + port
+            host = "127.0.0.1"
+          } else if (host === "[::1]") {
+            host = "::1"
+            ip = localAddress
           } else {
             ip = localAddress
           }
@@ -105,9 +110,8 @@ class Procs {
 
           if (pids.has(pid+"/"+port)) continue;
           pids.add(pid+"/"+port)
-          results.push({ port, pid, ip });
+          results.push({ port, pid, ip, host });
         } catch (e) {
-//            console.log("ERROR", e)
         }
       } else {
 //          if (!/LISTEN/.test(line)) continue;
@@ -123,27 +127,30 @@ class Procs {
 
           const match = line.match(/([^\s]+:\d+)\s/);
           const localAddress = match?.[1];
-
-          const chunks = localAddress.split(":")
-          const port = chunks.pop()
-          let ip = chunks.pop()
-          if (!ip || ip === "*") {
+          const chunks = /(.+):([0-9]+)/.exec(localAddress)
+          let host = chunks[1]
+          let port = chunks[2]
+          let ip
+          if (host === "*") {
             ip = "127.0.0.1:" + port
+            host = "127.0.0.1"
+          } else if (host === "[::1]") {
+            host = "::1"
+            ip = localAddress
           } else {
             ip = localAddress
           }
 
           if (pids.has(pid+"/"+port)) continue;
           pids.add(pid+"/"+port)
-          if (pid && port) results.push({ port, pid, ip });
+          if (pid && port) results.push({ port, pid, ip, host });
         } catch (e) {
-//            console.log("Error", e)
         }
       }
     }
-    const http_check = await Promise.all(results.map(({ port }) => {
+    const http_check = await Promise.all(results.map(({ host, port }) => {
       return limit(() => {
-        return this.isHttp(port) 
+        return this.isHttp(host, port)
       })
     }))
     let filtered = []