pinokiod 3.28.0 → 3.30.0

package/kernel/api/index.js

@@ -1176,7 +1176,6 @@ class Api {
 //        console.log("kernel.refresh after step")
 //        this.kernel.refresh(true)
       } catch (e) {
-        console.log("FUDKDFKSDFLJSDFHSDFLKHSDLFHSDFH", e)
         ondata({ raw: e.toString() })
       }
     }, concurrency)

package/kernel/bin/caddy.js

@@ -50,6 +50,8 @@ class Caddy {
         })
       })
       console.log("kernel.refresh bin.caddy.start")
+      this.kernel.peer.announce()
+      console.log("announced to peers")
 //      this.kernel.refresh(true)
     }
   }

package/kernel/index.js

@@ -292,6 +292,7 @@ class Kernel {
   }
   async refresh(notify_peers) {
     const ts = Date.now()
+//    console.time("Total " + ts)
     let network_active = await this.network_active()
     if (!network_active) {
       return
@@ -307,23 +308,12 @@ class Kernel {
       // 1. get the process list
 //      console.time("> 1. Proc Refresh"+ts)
       await this.processes.refresh()
-
-      // diff check
-      let new_config = JSON.stringify(this.processes.info)
-//      if (this.old_config !== new_config) {
-//        console.log("Proc config has changed")
-//        console.log("old", this.old_config)
-//        console.log("new", new_config)
-//      } else {
-//        console.log("Proc config is the same")
-//      }
-      this.old_config = new_config
-
 //      console.timeEnd("> 1. Proc Refresh"+ts)
 
       // 2. refresh peer info to reflect the proc info
 //      console.time("> 2. Peer Refresh"+ts)
-      await this.peer.refresh()
+      //await this.peer.refresh()
+      await this.peer.refresh_host(this.peer.host)
 //      console.timeEnd("> 2. Peer Refresh"+ts)
 
       // 3. load custom routers from ~/pinokio/network
@@ -336,18 +326,6 @@ class Kernel {
       await this.router.local()
 //      console.timeEnd("> 4. Router Local"+ts)
 
-      // 5. refresh peer info to reflect the updated router info
-//      console.time("> 5. Peer Refresh"+ts)
-      await this.peer.refresh()
-//      console.timeEnd("> 5. Peer Refresh"+ts)
-
-      // 6. tell peers to refresh
-      if (notify_peers) {
-//        console.time("> 6. Peer Notify Peers"+ts)
-        await this.peer.notify_peers()
-//        console.timeEnd("> 6. Peer Notify Peers"+ts)
-      }
-
       // 7. update remote router
 //      console.time("> 7. Router Remote"+ts)
       await this.router.remote()
@@ -355,14 +333,33 @@ class Kernel {
 
       await this.router.custom_domain()
 
+      this.router.fallback()
+
       // 8. update caddy config
 //      console.time("> 8. Router Update"+ts)
       await this.router.update()
 //      console.timeEnd("> 8. Router Update"+ts)
 
+      // 6. tell peers to refresh
+      let changed
+//      console.time("> 9. notify if changed")
+      let new_config = JSON.stringify(await this.peer.current_host())
+      if (this.old_config !== new_config) {
+        console.log("Proc config has changed. update router.")
+        changed = true
+      } else {
+//        console.log("Proc config is the same")
+        changed = false
+      }
+      this.old_config = new_config
+      if (changed) {
+        await this.peer.notify_refresh()
+      }
+//      console.timeEnd("> 9. notify if changed")
       // 9. announce self to the peer network
       this.peer.announce()
     }
+//    console.timeEnd("Total " + ts)
   }
   async clearLog(group) {
     let relativePath = path.relative(this.homedir, group)

package/kernel/peer.js

@@ -7,11 +7,13 @@ class PeerDiscovery {
     this.kernel = kernel
     this.port = port;
     this.message = Buffer.from(message);
+    this.kill_message = Buffer.from("kill")
     this.interval = interval;
     this.peers = new Set();
     this.host = this._getLocalIPAddress()
     this.default_port = 42000
     this.peers.add(this.host)
+    this.router_info_cache = {}
 //    this.start();
   }
   stop() {
@@ -20,6 +22,17 @@ class PeerDiscovery {
       this.socket.close()
     }
   }
+  async check_peers () {
+    for(let host of Array.from(this.peers)) {
+      if (this.host !== host) {
+        let result = await this._refresh(host)
+        if (!result) {
+          this.peers.delete(host)
+          delete this.info[host]
+        }
+      }
+    }
+  }
   announce() {
     if (this.socket) {
       this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
@@ -69,11 +82,19 @@ class PeerDiscovery {
       this.socket = dgram.createSocket('udp4');
       this.socket.on('message', (msg, rinfo) => {
         const ip = rinfo.address;
+        let str = msg.toString()
+        let kill_message = this.kill_message.toString()
+        if (str.startsWith(kill_message + " ") && this._isLocalLAN(ip)) {
+          let host = str.split(" ")[1]
+          console.log({ host })
+          this.kill(host)
+        }
         if (msg.toString() === this.message.toString() && this._isLocalLAN(ip)) {
           if (!this.peers.has(ip)) {
             console.log(`Discovered peer: ${ip}`);
             this.peers.add(ip);
-            this.refresh()
+//            this.refresh()
+//            this.notify_refresh()
           }
         }
       });
@@ -114,6 +135,25 @@ class PeerDiscovery {
       return null
     }
   }
+  async notify_refresh() {
+    // notify all peers of the current host info
+    if (this.info) {
+      let info = this.info[this.host]
+      for(let host of Array.from(this.peers)) {
+        if (this.host !== host) {
+          try {
+            let endpoint = `http://${host}:${this.default_port}/pinokio/peer/refresh`
+            let res = await axios.post(endpoint, info, {
+              timeout: 2000
+            })
+            return res.data
+          } catch (e) {
+            return null
+          }
+        }
+      }
+    }
+  }
   async _broadcast(host) {
     try {
       let res = await axios.post(`http://${host}:${this.default_port}/pinokio/peer/refresh`, {
@@ -133,19 +173,194 @@ class PeerDiscovery {
       return res
     }
   }
-  current_host() {
+  async proc_info(proc) {
+    let title
+    let description
+    let http_icon
+    let https_icon
+    let icon
+    let iconpath
+    let appname
+    if (proc.external_router) {
+      // try to get icons from pinokio
+      for(let router of proc.external_router) {
+        // replace the root domain: facefusion-pinokio.git.x.localhost => facefusion-pinokio.git
+        let pattern = `.${this.name}.localhost`
+        if (router.endsWith(pattern)) {
+          let name = router.replace(pattern, "")
+          appname = name
+          let api_path = this.kernel.path("api", name)
+          let exists = await this.kernel.exists(api_path)
+          if (exists) {
+            let meta = await this.kernel.api.meta(name)
+            if (meta.icon) {
+              icon = meta.icon
+              iconpath = meta.iconpath
+            }
+            if (meta.title) {
+              title = meta.title
+            }
+            if (meta.description) {
+              description = meta.description
+            }
+          }
+        }
+      }
+    }
+    // if not an app running inside pinokio, try to fetch and infer the favicon
+    if (icon) {
+      http_icon = `http://${this.host}:42000${icon}`;
+      //https_icon = `https://${appname}.${this.name}.localhost/${iconpath}?raw=true`
+      https_icon = `https://pinokio.${this.name}.localhost/raw/${appname}/${iconpath}`
+    } else {
+      for(let protocol of ["https", "http"]) {
+        if (protocol === "https") {
+          if (proc.external_router.length > 0) {
+            let favicon = await this.kernel.favicon.get("https://" + proc.external_router[0])
+            if (favicon) {
+              https_icon = favicon
+            }
+          }
+        } else {
+          if (proc.external_ip) {
+            let favicon = await this.kernel.favicon.get("http://" + proc.external_ip)
+            if (favicon) {
+              http_icon = favicon
+            }
+          }
+        }
+      }
+    }
+    return {
+      title, description, http_icon, https_icon, icon
+    }
+  }
+  async router_info() {
+    try {
+      let processes = []
+      if (this.info[this.host]) {
+        let procs = this.info[this.host].proc
+        let router = this.info[this.host].router
+        let port_mapping = this.info[this.host].port_mapping
+        for(let proc of procs) {
+          let pid = proc.pid
+          let d = Date.now()
+          let chunks = proc.ip.split(":")
+          let internal_port = chunks[chunks.length-1]
+          let internal_host = chunks.slice(0, chunks.length-1).join(":")
+          let external_port = port_mapping[internal_port]
+          let merged
+          let external_ip
+          if (external_port) {
+            external_ip = `${this.host}:${external_port}`
+          }
+          let info = {
+            external_router: router[external_ip] || [],
+            internal_router: router[proc.ip] || [],
+            external_ip,
+            external_port: parseInt(external_port),
+            internal_port: parseInt(internal_port),
+            ...proc,
+          }
+          let cached = this.router_info_cache[pid]
+          let cached_str = JSON.stringify(cached)
+          let info_str = JSON.stringify(info)
+          if (cached && cached_str === info_str) {
+            // nothing has changed. use the cached version
+            processes.push(cached)
+          } else {
+            // something has changed, refresh
+            let proc_info = await this.proc_info(info)
+            info = { ...proc_info, ...info }
+            this.router_info_cache[pid] = info
+            processes.push(info)
+          }
+        }
+      }
+      processes.sort((a, b) => {
+        return b.external_port-a.external_port
+      })
+      return processes
+    } catch (e) {
+      console.log("ERROR", e)
+      return []
+    }
+  }
+  async installed() {
+    let folders = await fs.promises.readdir(this.kernel.path("api"))
+    let installed = []
+    for(let folder of folders) {
+      let meta = await this.kernel.api.meta(folder)
+      /*
+      meta := {
+        title,
+        icon,
+        description,
+      }
+      */
+      let http_icon = null
+      let https_icon = null
+      let http_href = null
+      let https_href = null
+      if (meta && !meta.init_required) {
+        if (meta.title) {
+          if (meta.icon) {
+            http_icon = `http://${this.host}:42000${meta.icon}`;
+            //https_icon = `https://${folder}.${this.name}.localhost/${meta.iconpath}?raw=true`
+            https_icon = `https://pinokio.${this.name}.localhost/raw/${folder}/${meta.iconpath}`
+          }
+          //https_href = `https://${folder}.${this.name}.localhost`
+          https_href = `https://pinokio.${this.name}.localhost/p/${folder}`
+          http_href = `http://${this.host}:42000/p/${folder}`
+          installed.push({
+            folder,
+            http_icon,
+            https_icon,
+            https_href,
+            http_href,
+            ...meta 
+          })
+        }
+      }
+    }
+    return installed
+  }
+  async current_host() {
+    let d = Date.now()
+    let router_info = await this.router_info()
+    let installed = await this.installed()
     return {
       home: this.kernel.homedir,
       arch: this.kernel.arch,
       platform: this.kernel.platform,
-      name: this.kernel.peer.name,
-      host: this.kernel.peer.host,
+      name: this.name,
+      host: this.host,
       port_mapping: this.kernel.router.port_mapping,
       proc: this.kernel.processes.info,
       router: this.kernel.router.published(),
+      router_info,
+      installed,
       memory: this.kernel.memory
     }
   }
+  refresh_info(info) {
+    this.info[info.host] = info
+  }
+  async refresh_host(host) {
+    this.refreshing = true
+    if (!this.info) {
+      this.info = {}
+    }
+
+    let info = await this._refresh(host)
+    if (info) {
+      this.info[host] = {
+        host,
+        ...info
+      }
+    }
+    this.refreshing = false
+  }
   // refresh peer info
   async refresh(peers) {
 //    if (this.active) {
@@ -154,7 +369,9 @@ class PeerDiscovery {
       if (peers) {
         refresh_peers = peers
       } else {
-        this.info = {}
+        if (!this.info) {
+          this.info = {}
+        }
         refresh_peers = Array.from(this.peers)
       }
       let peer_info = await Promise.all(refresh_peers.map((host) => {
@@ -167,6 +384,14 @@ class PeerDiscovery {
             host: peer.host,
             ...peer
           }
+        } else {
+//          let host = refresh_peers[i]
+//          console.log(`remove peer ${host}`)
+//          delete this.info[host]
+//          this.peers.delete(host)
+//          console.log("after removing")
+//          console.log("info", this.info)
+//          console.log("peers", this.peers)
         }
       }
       this.refreshing = false
@@ -176,29 +401,6 @@ class PeerDiscovery {
   _isLocalLAN(ip) {
     return ip.startsWith('192.168.') || ip.startsWith('10.') || (ip.startsWith('172.') && is172Private(ip));
   }
-  //_getLocalIPAddress() {
-  //  const interfaces = os.networkInterfaces();
-  //  for (const ifaceList of Object.values(interfaces)) {
-  //    for (const iface of ifaceList) {
-  //      console.log({ iface })
-  //      if (iface.family === 'IPv4' && !iface.internal) {
-  //        const ip = iface.address;
-  //        if (
-  //          ip.startsWith('10.') ||
-  //          ip.startsWith('192.168.') ||
-  //          (ip.startsWith('172.') && is172Private(ip))
-  //        ) {
-  //          return ip;
-  //        }
-  //      }
-  //    }
-  //  }
-  //  return null;
-  //  function is172Private(ip) {
-  //    const secondOctet = parseInt(ip.split('.')[1], 10);
-  //    return secondOctet >= 16 && secondOctet <= 31;
-  //  }
-  //}
   _getLocalIPAddress() {
     const interfaces = os.networkInterfaces();
     for (const ifaceList of Object.values(interfaces)) {

package/kernel/procs.js

@@ -10,28 +10,10 @@ const cls = isWin ? 'cls' : 'clear'
 const net = require('net');
 class Procs {
   constructor (kernel) {
-//    console.log("Initializing procs")
     this.kernel = kernel
     this.cache = {}
   }
   async isHttp(port) {
-  //async isHttp(localAddress) {
-//    if (this.cache.hasOwnProperty(localAddress)) {
-////      console.log("Use cached", localAddress)
-//      return this.cache[localAddress]
-//    }
-//    console.log("Not cached", localAddress)
-    //try {
-    //  //await axios.head(`http://${localAddress}`, { timeout: 3000 });
-    //  await axios.get(`http://${localAddress}`, { timeout: 3000 });
-    //  this.cache[localAddress] = true
-    //  return true;
-    //} catch (err) {
-    //  console.log("HEAD ERROR",{ localAddress, err })
-    //  this.cache[localAddress] = false
-    //  return false;
-    //}
-
     // ignore caddy
     if (parseInt(port) === 2019) {
       return false
@@ -79,7 +61,6 @@ class Procs {
     })
   }
   emit(id, cmd) {
-//    console.log("emit", { id, cmd })
     setTimeout(() => {
       this.kernel.shell.emit({
         emit: cmd + "\r\n" + cls,
@@ -88,7 +69,6 @@ class Procs {
     }, 10)
   }
   async get_pids (stdout) {
-//    console.log("get_pids size", stdout.length)
     const results = [];
     let pids = new Set()
     let s = stdout.trim()
@@ -153,11 +133,6 @@ class Procs {
             ip = localAddress
           }
 
-//          let isHttp = await this.isHttp(ip)
-//          if (!isHttp) continue;
-
-          //const portMatch = line.match(/:(\d+)\s/);
-          //const port = portMatch?.[1];
           if (pids.has(pid+"/"+port)) continue;
           pids.add(pid+"/"+port)
           if (pid && port) results.push({ port, pid, ip });
@@ -166,14 +141,11 @@ class Procs {
         }
       }
     }
-//    console.timeEnd("###### Line parsing")
-//    console.time("########## http_check")
     const http_check = await Promise.all(results.map(({ port }) => {
       return limit(() => {
         return this.isHttp(port) 
       })
     }))
-//    console.timeEnd("########## http_check")
     let filtered = []
     for(let i=0; i<http_check.length; i++) {
       if (http_check[i]) {
@@ -184,53 +156,7 @@ class Procs {
   }
   getPortPidList(cb) {
     const cmd = isWin ? 'netstat -ano -p tcp' : 'lsof -nP -iTCP -sTCP:LISTEN';
-    /*
-//    if (this.portPidList) {
-//      cb(this.portPidList)
-//      return
-//    }
-    let id = "Procs.getPortPidList"
-    let sh = this.kernel.shell.get(id)
-    this.port_cb = cb
-    this.port_running = true
-    this.d2 = Date.now()
-    console.time("Shell"+this.d2)
-    if (sh) {
-      this.emit(id, cmd)
-      console.time("lsof" + this.d2);
-    } else {
-      this.kernel.exec({
-        id,
-        conda: {
-          skip: true,
-        },
-        onready: () => {
-          this.emit(id, cmd)
-        },
-        input: true
-      }, (e) => {
-        console.log(">>>>>> all e.state", e.state)
-        if (e.state && e.state.includes(cls)) {
-          if (this.port_running) {
-            console.timeEnd("Shell"+this.d2)
-            console.log("e.state", e.state)
-            this.port_running = false
-            this.newline(id)
-            let d = Date.now()
-            console.time("get_pids" + d)
-            this.get_pids(e.state).then((pids) => {
-              console.timeEnd("get_pids" + d)
-              this.portPidList = pids
-              this.port_cb(pids)
-            })
-          }
-        }
-      }).then((result) => {
-        this.port_running = false
-        console.log("Exec Finished",  {result})
-      })
-    }
-    */
+    let d = Date.now()
     exec(cmd, { maxBuffer: 10 * 1024 * 1024 }, async (err, stdout) => {
       this.get_pids(stdout).then((pids) => {
         cb(pids)
@@ -238,7 +164,6 @@ class Procs {
     });
   }
   get_name(stdout) {
-//    console.log("get_name size", stdout.length)
     const lines = stdout.trim().split('\n');
     const map = {};
     lines.forEach(line => {
@@ -265,7 +190,6 @@ class Procs {
         let exists = this.port_map["" + pid]
         if (!exists) {
           cached = false 
-          console.log("Didn't exist", { pid, port, exists })
           break;
         }
       }
@@ -275,46 +199,6 @@ class Procs {
       return
     }
     const cmd = isWin ? 'tasklist /fo csv /nh' : 'ps -Ao pid,comm';
-    /*
-    let id = "Procs.getPidToNameMap"
-    let sh = this.kernel.shell.get(id)
-    this.pid_cb = cb
-    this.pid_running = true
-    if (sh) {
-      this.emit(id, cmd)
-    } else {
-      this.kernel.exec({
-        id,
-        conda: {
-          skip: true,
-        },
-        onready: () => {
-//          console.log("ON READY")
-          this.emit(id, cmd)
-        },
-        input: true
-      }, async (e) => {
-        if (e.state && e.state.includes(cls)) {
-          if (this.pid_running) {
-            this.pid_running = false
-            this.newline(id)
-//            console.log("GET MAP")
-            let map = this.get_name(e.state)
-            if (!this.port_map) {
-              this.port_map = {}
-            }
-            for(let key in map) {
-              this.port_map[key] = map[key]
-            }
-            this.pid_cb(this.port_map)
-          }
-        }
-      }).then((result) => {
-        this.pid_running = false
-        console.log("Exec Finished",  {result})
-      })
-    }
-    */
     exec(cmd, { maxBuffer: 10 * 1024 * 1024 }, async (err, stdout) => {
       let map = this.get_name(stdout)
       if (!this.port_map) {
@@ -331,21 +215,8 @@ class Procs {
     this.refreshing = true
     let ts = Date.now()
     let list = await new Promise((resolve, reject) => {
-//      console.time(">>>>>>>>GET PORTS " + ts)
-//      console.log("get ports")
       this.getPortPidList((portPidList) => {
-//        console.log("done: get ports")
-//        console.log({ portPidList })
-//        console.timeEnd(">>>>>>>>GET PORTS " + ts)
-//        console.time(">>>>>>> GET PIDS " + ts)
-        // if there's any new port, run getPidToNameMap
-
-
-//        console.log("getPid")
         this.getPidToNameMap(portPidList, (pidToName) => {
-//          console.log("done getPid")
-          
-//          console.timeEnd(">>>>>>> GET PIDS " + ts)
           let list = portPidList.map(({ port, pid, ip }) => {
             const fullname = pidToName[pid] || 'Unknown';
             const name = fullname.split(path.sep).pop()
@@ -357,7 +228,6 @@ class Procs {
               return { port, pid , name, fullname, ip }
             }
           }).filter((x) => { return x })
-//          console.log("LIST", JSON.stringify(list, null, 2))
           resolve(list)
         })
       })

package/kernel/prototype.js

@@ -77,15 +77,12 @@ class Proto {
     await fs.promises.rm(this.kernel.path("prototype"), { recursive: true })
   }
   async create(req, ondata) {
-    console.log("REQ", req)
     try {
       let projectType = req.params.projectType
       let startType = req.params.cliType || req.params.startType
-      console.log({ projectType, startType })
 
       let cwd = req.cwd
       let name = req.params.name
-//      let name = req.name
       let payload = {}
       payload.cwd = path.resolve(cwd, name)
       payload.input = req.params
@@ -100,8 +97,6 @@ class Proto {
       let mod_path = this.kernel.path("prototype/system", projectType, startType)
       let mod = await this.kernel.require(mod_path)
 
-      console.log({ mod_path, projectType, startType })
-
       await mod(payload, ondata, this.kernel)
 
       // copy readme
@@ -120,7 +115,6 @@ class Proto {
     }
   }
   async readme(proto) {
-    console.log("proto.readme", proto)
     let readme_path = path.resolve(this.kernel.homedir, "prototype", proto, "README.md")
     let readme
     try {
@@ -128,7 +122,6 @@ class Proto {
     } catch (e) {
       readme = ""
     }
-    console.log({ readme_path, readme })
     return marked.parse(readme)
   }
 }