pinokiod 3.15.0 → 3.15.2

package/kernel/api/index.js

@@ -233,7 +233,8 @@ class Api {
       id: req.params.id || requestPath,
       type: "disconnect"
     })
-    this.kernel.refresh(true)
+//    console.log("kernel.refresh api.stop")
+//    this.kernel.refresh(true)
     return true
   }
   async startScripts() {
@@ -1153,7 +1154,8 @@ class Api {
             })
           }
         }
-        this.kernel.refresh(true)
+//        console.log("kernel.refresh after step")
+//        this.kernel.refresh(true)
       } catch (e) {
         ondata({ raw: e.toString() })
       }

package/kernel/bin/caddy.js

@@ -1,12 +1,13 @@
 const axios = require('axios')
 const fs = require('fs')
+const semver = require('semver')
 
 class Caddy {
   cmd() {
     if (this.kernel.platform === "win32") {
-      return "caddy"
+      return "caddy=2.9.1"
     } else {
-      return "caddy nss"
+      return "caddy=2.9.1 nss"
     }
   }
   async running() {
@@ -44,7 +45,8 @@ class Caddy {
           }
         })
       })
-      this.kernel.refresh(true)
+      console.log("kernel.refresh bin.caddy.start")
+//      this.kernel.refresh(true)
     }
   }
   async install(req, ondata, kernel, id) {
@@ -101,14 +103,25 @@ class Caddy {
     }
   }
   async installed() {
-    let e = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/pki/authorities/local/root.crt"))
-    if (e) {
-      if (this.kernel.platform === "win32") {
-        return this.kernel.bin.installed.conda.has("caddy")
+    try {
+      let version = this.kernel.bin.installed.conda_versions.caddy
+      let coerced = semver.coerce(version)
+      let requirement = "<2.10.0"
+      let satisfied = semver.satisfies(coerced, requirement)
+      if (!satisfied) {
+        return false 
+      }
+      let e = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/pki/authorities/local/root.crt"))
+      if (e) {
+        if (this.kernel.platform === "win32") {
+          return this.kernel.bin.installed.conda.has("caddy")
+        } else {
+          return this.kernel.bin.installed.conda.has("caddy") && this.kernel.bin.installed.conda.has("nss")
+        }
       } else {
-        return this.kernel.bin.installed.conda.has("caddy") && this.kernel.bin.installed.conda.has("nss")
+        return false
       }
-    } else {
+    } catch (e) {
       return false
     }
   }

package/kernel/bin/setup.js

@@ -159,7 +159,6 @@ module.exports = {
       { name: "caddy", },
       { name: "py", },
     ])
-    console.log("Setup.network", requirements)
     return {
       icon: "fa-solid fa-wifi",
       title: "Network",
@@ -187,7 +186,6 @@ module.exports = {
       { name: "caddy", },
       { name: "py", },
     ])
-    console.log("Setup.connect", requirements)
     return {
       icon: "fa-solid fa-wifi",
       title: "API Connect",

package/kernel/index.js

@@ -262,57 +262,87 @@ class Kernel {
   log(data, group, info) {
     this.log_queue.push({ data, group, info })
   }
-  async refresh(notify_peers) {
-    let env = await Environment.get(this.homedir)
-    let peer_active = true
-    // if PINOKIO_NETWORK_SHARE is 0 or false, turn it off
-    // By default it should be on.
-    // It's off if the value is set, and it's 0 or false
+  async should_refresh() {
+    if (this.refresh_cached) {
+      return this.cached_refresh_value
+    } else {
+      let env = await Environment.get(this.homedir)
+      let peer_active = true
+      // if PINOKIO_NETWORK_SHARE is 0 or false, turn it off
+      // By default it should be on.
+      // It's off if the value is set, and it's 0 or false
+
+      //if (env && env.PINOKIO_NETWORK_ACTIVE && (env.PINOKIO_NETWORK_ACTIVE==="1" || env.PINOKIO_NETWORK_ACTIVE.toLowerCase()==="true")) {
+      if (env && env.PINOKIO_NETWORK_ACTIVE && (env.PINOKIO_NETWORK_ACTIVE==="0" || env.PINOKIO_NETWORK_ACTIVE.toLowerCase()==="false")) {
+        peer_active = false
+      }
+      let https_active = true
+      //if (env && env.PINOKIO_HTTPS_ACTIVE && (env.PINOKIO_HTTPS_ACTIVE==="1" || env.PINOKIO_HTTPS_ACTIVE.toLowerCase()==="true")) {
+      if (env && env.PINOKIO_HTTPS_ACTIVE && (env.PINOKIO_HTTPS_ACTIVE==="0" || env.PINOKIO_HTTPS_ACTIVE.toLowerCase()==="false")) {
+        //https_active = true
+        https_active = false 
+      }
+  //    console.log("kernel.refresh", { active, notify_peers })
 
-    //if (env && env.PINOKIO_NETWORK_ACTIVE && (env.PINOKIO_NETWORK_ACTIVE==="1" || env.PINOKIO_NETWORK_ACTIVE.toLowerCase()==="true")) {
-    if (env && env.PINOKIO_NETWORK_ACTIVE && (env.PINOKIO_NETWORK_ACTIVE==="0" || env.PINOKIO_NETWORK_ACTIVE.toLowerCase()==="false")) {
-      peer_active = false
-    }
-    let https_active = true
-    //if (env && env.PINOKIO_HTTPS_ACTIVE && (env.PINOKIO_HTTPS_ACTIVE==="1" || env.PINOKIO_HTTPS_ACTIVE.toLowerCase()==="true")) {
-    if (env && env.PINOKIO_HTTPS_ACTIVE && (env.PINOKIO_HTTPS_ACTIVE==="0" || env.PINOKIO_HTTPS_ACTIVE.toLowerCase()==="false")) {
-      //https_active = true
-      https_active = false 
+      let caddy_installed = await this.bin.check_installed({ name: "caddy" })
+      this.refresh_cached = true
+      this.cached_refresh_value = caddy_installed && https_active
+      return this.cached_refresh_value
     }
-//    console.log("kernel.refresh", { active, notify_peers })
+  }
+  async refresh(notify_peers) {
+    const ts = Date.now()
+    let should_refresh = await this.should_refresh()
+    if (should_refresh) {
 
-    let caddy_installed = await this.bin.check_installed({ name: "caddy" })
 
-    // 1. https
-    // 2. https + local share
-//    console.log("caddy installed?", caddy_installed)
-    if (caddy_installed && https_active) {
+      let ts = Date.now()
+      if (this.processes.refreshing) {
+        // process list refreshing. try again later
+        return
+      }
 
       // 1. get the process list
+//      console.time("> 1. Proc Refresh"+ts)
       await this.processes.refresh()
+//      console.timeEnd("> 1. Proc Refresh"+ts)
 
       // 2. refresh peer info to reflect the proc info
+//      console.time("> 2. Peer Refresh"+ts)
       await this.peer.refresh()
+//      console.timeEnd("> 2. Peer Refresh"+ts)
 
       // 3. load custom routers from ~/pinokio/network
+//      console.time("> 3. Router Init"+ts)
       await this.router.init()
+//      console.timeEnd("> 3. Router Init"+ts)
 
       // 4. set current local host router info
+//      console.time("> 4. Router Local"+ts)
       await this.router.local()
+//      console.timeEnd("> 4. Router Local"+ts)
 
       // 5. refresh peer info to reflect the updated router info
+//      console.time("> 5. Peer Refresh"+ts)
       await this.peer.refresh()
+//      console.timeEnd("> 5. Peer Refresh"+ts)
 
       // 6. tell peers to refresh
       if (notify_peers) {
+//        console.time("> 6. Peer Notify Peers"+ts)
         await this.peer.notify_peers()
+//        console.timeEnd("> 6. Peer Notify Peers"+ts)
       }
 
       // 7. update remote router
+//      console.time("> 7. Router Remote"+ts)
       await this.router.remote()
+//      console.timeEnd("> 7. Router Remote"+ts)
 
       // 8. update caddy config
+//      console.time("> 8. Router Update"+ts)
       await this.router.update()
+//      console.timeEnd("> 8. Router Update"+ts)
 
       // 9. announce self to the peer network
       this.peer.announce()
@@ -549,7 +579,7 @@ class Kernel {
     this.pipe = new Pipe(this)
     this.proto = new Proto(this)
     this.plugin = new Plugin(this)
-    this.processes = new Procs()
+    this.processes = new Procs(this)
     this.kv = new KV(this)
     this.cloudflare = new Cloudflare()
     this.peer = new Peer()
@@ -768,11 +798,16 @@ class Kernel {
             return this.api.get_proxy_url("/proxy", port)
           },
         })
+//        setTimeout(() => {
+//          this.refresh()    
+//        }, 3000)
 
         // refresh every 5 second
-        if (this.refresh_interval) {
-          clearInterval(this.refresh_interval)
-        }
+//        if (this.refresh_interval) {
+//          clearInterval(this.refresh_interval)
+//        }
+        this.refresh_cached = false
+        this.cached_refresh_value = false
         this.refresh_interval = setInterval(() => {
           this.refresh()    
         }, 5000)

package/kernel/procs.js

@@ -1,11 +1,14 @@
 const os = require('os')
 const path = require('path')
 const axios = require('axios')
-const { fork, exec } = require('child_process');
+const { fork, exec, spawn } = require('child_process');
 const platform = os.platform();
+const isWin = platform === 'win32';
+const cls = isWin ? 'cls' : 'clear'
 class Procs {
-  constructor () {
-    console.log("Initializing procs")
+  constructor (kernel) {
+//    console.log("Initializing procs")
+    this.kernel = kernel
     this.cache = {}
   }
   async isHttp(localAddress) {
@@ -21,129 +24,235 @@ class Procs {
       return false;
     }
   }
-  getPortPidList(callback) {
-    const isWin = platform === 'win32';
-    const cmd = isWin ? 'netstat -ano -p tcp' : 'lsof -nP -iTCP -sTCP:LISTEN';
-    exec(cmd, { maxBuffer: 10 * 1024 * 1024 }, async (err, stdout) => {
-      if (err) return callback(err);
-
-      const results = [];
-      let pids = new Set()
-      let s = stdout.trim()
-//      console.log(">> getPortPidList", s)
-      const lines = s.split('\n');
+  emit(id, cmd) {
+//    console.log("emit", { id, cmd })
+    this.kernel.shell.emit({
+      emit: "\r\n",
+      id,
+    })
+    this.kernel.shell.emit({
+      emit: cmd + "\r\n" + cls,
+      id,
+    })
+  }
+  async get_pids (stdout) {
+//    console.log("get_pids size", stdout.length)
+    const results = [];
+    let pids = new Set()
+    let s = stdout.trim()
+    const lines = s.split('\n');
 
-      for(let line of lines) {
-        if (isWin) {
-          // Skip headers
-          try {
-            if (!line.startsWith('  TCP')) continue;
-            const parts = line.trim().split(/\s+/);
-            const [ , localAddress, , state, pid ] = parts;
+    for(let line of lines) {
+      if (isWin) {
+        // Skip headers
+        try {
+          if (!line.startsWith('  TCP')) continue;
+          const parts = line.trim().split(/\s+/);
+          const [ , localAddress, , state, pid ] = parts;
 
-            let pid_int = parseInt(pid)
-            if (pid_int === 0 || pid_int === 4) {
-              // pid 0 => killed processes => irrelevant
-              // pid 4 => system process => irrelevant
-              continue
-            }
+          let pid_int = parseInt(pid)
+          if (pid_int === 0 || pid_int === 4) {
+            // pid 0 => killed processes => irrelevant
+            // pid 4 => system process => irrelevant
+            continue
+          }
 
-            //if (state !== 'LISTENING') continue;
-            const chunks = localAddress.split(":")
-            const port = chunks.pop()
-            let ip = chunks.pop()
-            if (!ip || ip === "*") {
-              ip = "127.0.0.1:" + port
-            } else {
-              ip = localAddress
-            }
+          //if (state !== 'LISTENING') continue;
+          const chunks = localAddress.split(":")
+          const port = chunks.pop()
+          let ip = chunks.pop()
+          if (!ip || ip === "*") {
+            ip = "127.0.0.1:" + port
+          } else {
+            ip = localAddress
+          }
 
-            let isHttp = await this.isHttp(ip)
-            if (!isHttp) continue
+          let isHttp = await this.isHttp(ip)
+          if (!isHttp) continue
 
-            if (pids.has(pid+"/"+port)) continue;
-            pids.add(pid+"/"+port)
-            results.push({ port, pid, ip });
-          } catch (e) {
+          if (pids.has(pid+"/"+port)) continue;
+          pids.add(pid+"/"+port)
+          results.push({ port, pid, ip });
+        } catch (e) {
 //            console.log("ERROR", e)
-          }
-        } else {
+        }
+      } else {
 //          if (!/LISTEN/.test(line)) continue;
-          try {
-            const parts = line.trim().split(/\s+/);
-            const pid = parts[1];
-            let pid_int = parseInt(pid)
-            if (pid_int === 0 || pid_int === 4) {
-              // pid 0 => killed processes => irrelevant
-              // pid 4 => system process => irrelevant
-              continue
-            }
+        try {
+          const parts = line.trim().split(/\s+/);
+          const pid = parts[1];
+          let pid_int = parseInt(pid)
+          if (pid_int === 0 || pid_int === 4) {
+            // pid 0 => killed processes => irrelevant
+            // pid 4 => system process => irrelevant
+            continue
+          }
 
-            const match = line.match(/([^\s]+:\d+)\s/);
-            const localAddress = match?.[1];
+          const match = line.match(/([^\s]+:\d+)\s/);
+          const localAddress = match?.[1];
 
-            const chunks = localAddress.split(":")
-            const port = chunks.pop()
-            let ip = chunks.pop()
-            if (!ip || ip === "*") {
-              ip = "127.0.0.1:" + port
-            } else {
-              ip = localAddress
-            }
+          const chunks = localAddress.split(":")
+          const port = chunks.pop()
+          let ip = chunks.pop()
+          if (!ip || ip === "*") {
+            ip = "127.0.0.1:" + port
+          } else {
+            ip = localAddress
+          }
 
-            let isHttp = await this.isHttp(ip)
-            if (!isHttp) continue;
+          let isHttp = await this.isHttp(ip)
+          if (!isHttp) continue;
 
-            //const portMatch = line.match(/:(\d+)\s/);
-            //const port = portMatch?.[1];
-            if (pids.has(pid+"/"+port)) continue;
-            pids.add(pid+"/"+port)
-            if (pid && port) results.push({ port, pid, ip });
-          } catch (e) {
+          //const portMatch = line.match(/:(\d+)\s/);
+          //const port = portMatch?.[1];
+          if (pids.has(pid+"/"+port)) continue;
+          pids.add(pid+"/"+port)
+          if (pid && port) results.push({ port, pid, ip });
+        } catch (e) {
 //            console.log("Error", e)
-          }
         }
       }
-      callback(null, results);
-    });
+    }
+    return results
   }
-  getPidToNameMap(callback) {
-    const isWin = platform === 'win32';
-    const cmd = isWin ? 'tasklist /fo csv /nh' : 'ps -Ao pid,comm';
-    exec(cmd, (err, stdout) => {
-      if (err) return callback(err);
-      const lines = stdout.trim().split('\n');
-      const map = {};
-      lines.forEach(line => {
-        if (isWin) {
-          const [name, pid] = line.split('","').map(s => s.replace(/(^"|"$)/g, ''));
+  getPortPidList(cb) {
+//    if (this.portPidList) {
+//      cb(this.portPidList)
+//      return
+//    }
+    const cmd = isWin ? 'netstat -ano -p tcp' : 'lsof -nP -iTCP -sTCP:LISTEN';
+    let id = "Procs.getPortPidList"
+    let sh = this.kernel.shell.get(id)
+    this.port_cb = cb
+    this.port_running = true
+    if (sh) {
+      this.emit(id, cmd)
+    } else {
+      this.kernel.exec({
+        id,
+        conda: {
+          skip: true,
+        },
+        onready: () => {
+//          console.log("ON READY")
+          this.emit(id, cmd)
+        },
+        input: true
+      }, (e) => {
+        if (e.state && e.state.includes(cls)) {
+          if (this.port_running) {
+            this.port_running = false
+//            console.log("GET PIDS")
+            this.get_pids(e.state).then((pids) => {
+              this.portPidList = pids
+              this.port_cb(pids)
+            })
+          }
+        }
+      }).then((result) => {
+        this.port_running = false
+        console.log("Exec Finished",  {result})
+      })
+    }
+//    exec(cmd, { maxBuffer: 10 * 1024 * 1024 }, async (err, stdout) => {
+//      if (err) return callback(err);
+//
+//      callback(null, results);
+//    });
+  }
+  get_name(stdout) {
+//    console.log("get_name size", stdout.length)
+    const lines = stdout.trim().split('\n');
+    const map = {};
+    lines.forEach(line => {
+      if (isWin) {
+        const [name, pid] = line.split('","').map(s => s.replace(/(^"|"$)/g, ''));
+        if (/^[0-9]+$/.test(pid)) {
+          // it is pid
           map[pid] = name;
-        } else {
-          const [pid, ...nameParts] = line.trim().split(/\s+/);
+        }
+      } else {
+        const [pid, ...nameParts] = line.trim().split(/\s+/);
+        if (/^[0-9]+$/.test(pid)) {
           map[pid] = nameParts.join(' ');
         }
-      });
-      callback(null, map);
+      }
     });
+    return map
+  }
+  getPidToNameMap(portPidList, cb) {
+    let cached = false
+    if (this.port_map) {
+      cached = true
+      for(let { pid, port } of portPidList) {
+        let exists = this.port_map["" + pid]
+//        console.log({ pid, exists })
+        if (!exists) {
+          cached = false 
+          break;
+        }
+      }
+    }
+    if (cached) {
+//      console.log("already cached")
+      cb(this.port_map)
+      return
+    }
+//    console.log("not cached")
+    const cmd = isWin ? 'tasklist /fo csv /nh' : 'ps -Ao pid,comm';
+    let id = "Procs.getPidToNameMap"
+    let sh = this.kernel.shell.get(id)
+    this.pid_cb = cb
+    this.pid_running = true
+    if (sh) {
+      this.emit(id, cmd)
+    } else {
+      this.kernel.exec({
+        id,
+        conda: {
+          skip: true,
+        },
+        onready: () => {
+//          console.log("ON READY")
+          this.emit(id, cmd)
+        },
+        input: true
+      }, async (e) => {
+        if (e.state && e.state.includes(cls)) {
+          if (this.pid_running) {
+//            console.log("GET MAP")
+            this.pid_running = false
+            let map = this.get_name(e.state)
+            this.port_map = map
+            this.pid_cb(map)
+          }
+        }
+      }).then((result) => {
+        this.pid_running = false
+        console.log("Exec Finished",  {result})
+      })
+    }
   }
   async refresh() {
     let map = {}
+    if (this.refreshing) {
+//      console.log("still refreshing. retry later") 
+      return
+    }
+    this.refreshing = true
+    let ts = Date.now()
     let list = await new Promise((resolve, reject) => {
-      this.getPortPidList((err, portPidList) => {
+//      console.time(">>>>>>>>GET PORTS " + ts)
+      this.getPortPidList((portPidList) => {
+//        console.timeEnd(">>>>>>>>GET PORTS " + ts)
+//        console.time(">>>>>>> GET PIDS " + ts)
 //        console.log({ portPidList })
-        if (err) {
-          console.log("getPortPidList Error", err)
-          reject(err)
-          return
-        }
 
-        this.getPidToNameMap((err, pidToName) => {
-          if (err) {
-            console.log("getPidToNameMap Error", err)
-            reject(err)
-            return
-          }
+        // if there's any new port, run getPidToNameMap
+
 
+        this.getPidToNameMap(portPidList, (pidToName) => {
+//          console.timeEnd(">>>>>>> GET PIDS " + ts)
           let list = portPidList.map(({ port, pid, ip }) => {
             const fullname = pidToName[pid] || 'Unknown';
             const name = fullname.split(path.sep).pop()
@@ -156,11 +265,12 @@ class Procs {
             }
           }).filter((x) => { return x })
           resolve(list)
-        });
-      });      
+        })
+      })
     })
     this.info = list
     this.map = map
+    this.refreshing = false
   }
 }
 module.exports = Procs

package/kernel/shell.js

@@ -349,6 +349,7 @@ class Shell {
     }
   }
   emit(message) {
+    //console.log("emit", { message, input: this.input, pty: this.ptyProcess })
     if (this.input) {
       if (this.ptyProcess) {
         if (message.length > 1024) {
@@ -938,6 +939,7 @@ class Shell {
           this.done = false
           this.ptyProcess = pty.spawn(this.shell, this.args, config)
           this.ptyProcess.onData((data) => {
+//            console.log("onData", { data })
             if (!this.done) {
               this.queue.push(data)
             }
@@ -988,6 +990,7 @@ class Shell {
     }
     this.vt.dispose()
     this.queue.killAndDrain()
+    console.log("KILL PTY", this.id)
     if (this.ptyProcess) {
       if (cb) {
         kill(this.ptyProcess.pid, "SIGKILL", true)
@@ -1025,7 +1028,7 @@ class Shell {
 //      id: this.id,
 //      type: "disconnect"
 //    })
-    this.kernel.refresh(true)
+//    this.kernel.refresh(true)
 
 
   }
@@ -1166,6 +1169,9 @@ ${cleaned}
         if (test) {
           if (test.length > 0) {
             this.ready = true
+            if (this.params && this.params.onready) {
+              this.params.onready()
+            }
             if (this.ptyProcess) {
               this.ptyProcess.write(`${this.cmd}${os.EOL}`)
 //              setTimeout(() => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "3.15.0",
+  "version": "3.15.2",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -3359,6 +3359,8 @@ class Server {
     this.app.post("/network/reset", ex(async (req, res) => {
       let caddy_path = this.kernel.path("cache/XDG_DATA_HOME/caddy")
       await rimraf(caddy_path)
+      let caddy_path2 = this.kernel.path("cache/XDG_CONFIG_HOME/caddy")
+      await rimraf(caddy_path2)
       res.json({ success: true })
     }))
     this.app.get("/network", ex(async (req, res) => {
@@ -4256,6 +4258,7 @@ class Server {
     }))
     this.app.post("/pinokio/peer/refresh", ex(async (req, res) => {
       // refresh and broadcast
+      console.log("kernel.refresh /pinokio/peer/refresh")
       await this.kernel.refresh()
       res.json({ success: true })
     }))