pinokiod 3.9.0 → 3.9.3

package/kernel/router/peer_peer_router.js

@@ -0,0 +1,26 @@
+const Common = require('./common')
+class PeerPeerRouter {
+  constructor(router) {
+    this.router = router
+    this.common = new Common(router)
+  }
+  handle(peer) {
+    for(let dial in peer.router) {
+      let matches = peer.router[dial]
+      let exposed_matches = matches.filter((m) => {
+        return m.endsWith(`${peer.name}.localhost`)
+      })
+      let chunks = dial.split(":")
+      let port = chunks[chunks.length-1]
+      let exposed_dial = `${peer.host}:${port}` // exposed_dial :- 192.168...
+      for(let match of exposed_matches) {
+        this.common.handle({
+          match,
+          dial: exposed_dial,
+          host: peer.host
+        })
+      }
+    }
+  }
+}
+module.exports = PeerPeerRouter

package/kernel/router/peer_port_router.js

@@ -0,0 +1,45 @@
+const path = require('path')
+const Processor = require('./processor')
+const Connector = require('./connector')
+class PeerPortRouter extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+    this.connector = new Connector(router)
+  }
+  handle(peer) {
+    let https_domains_mapping = this.router.info[peer.host]
+    for(let dial in https_domains_mapping) {
+      let parsed_dial = this.parse_ip(dial)
+      if (parsed_dial.host !== peer.host) {
+        // ONLY connect for 127.0.0.1 or localhost
+        // DO NOT connect if it's the actual host => because it will keep creating proxies forever recursively. only need to create proxies for localhost
+        let https_domains = https_domains_mapping[dial]
+
+        // 1. if https_domains is made up of one item <port>.localhost,
+        // add <port>.<account>.localhost
+        // 2. if https_domains is made up of two items <port>.localhost and <port>.<account>.localhost, don't do anything
+        for(let https_domain of https_domains) {
+          // insert the $account value right before the last part (1234.localhost => 1234.x.localhost)
+          let match
+          if (https_domain.endsWith(`${peer.name}.localhost`)) {
+            match = https_domain
+          } else {
+            let chunks = https_domain.split(".")
+            chunks.splice(chunks.length - 1, 0, peer.name)
+            match = chunks.join(".")
+          }
+          this.connector.handle({
+            match,
+            connector: {
+              host: peer.host,
+              port: parsed_dial.port
+            },
+            dial,
+          })
+        }
+      }
+    }
+  }
+}
+module.exports = PeerPortRouter

package/kernel/router/peer_variable_router.js

@@ -0,0 +1,46 @@
+const path = require('path')
+const Connector = require('./connector')
+const Processor = require('./processor')
+class PeerVariableRouter extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+    this.connector = new Connector(router)
+  }
+  handle(peer) {
+    for(let script_path in peer.memory.local) {
+      let local_variables = peer.memory.local[script_path]
+      for(let key in local_variables) {
+        let val = local_variables[key]
+        if (val.startsWith("http")) {
+          let dial = val.replace(/https?:\/\//, '')
+          let name = this.api_name(peer.platform, peer.home, script_path)
+          if (this.has_port(dial)) {
+            let parsed_dial = this.parse_ip(dial)
+            if (key === "url") {
+              this.connector.handle({
+                match: `${name}.${peer.name}.localhost`.toLowerCase(),
+                connector: {
+                  host: peer.host,
+                  port: parsed_dial.port
+                },
+                dial,
+                host: peer.host,
+              })
+            }
+            this.connector.handle({
+              match:`${key}.${name}.${peer.name}.localhost`.toLowerCase(),
+              connector: {
+                host: peer.host,
+                port: parsed_dial.port
+              },
+              dial,
+              host: peer.host,
+            })
+          }
+        }
+      }
+    }
+  }
+}
+module.exports = PeerVariableRouter

package/kernel/router/processor.js

@@ -0,0 +1,31 @@
+const path = require('path')
+class Processor {
+  has_port(str) {
+    let chunks = str.split(":")
+    return chunks.length === 2
+  }
+  api_name(platform, home, str) {
+    let api_path
+    let rel_path
+    let name
+    let _path
+    if (platform === 'win32') {
+      _path = path.win32
+      api_path = _path.resolve(home, "api")
+      rel_path = _path.relative(api_path, str)
+    } else {
+      _path = path.posix
+      api_path = _path.resolve(home, "api")
+      rel_path = _path.relative(api_path, str)
+    }
+    name = rel_path.split(_path.sep)[0]
+    return name
+  }
+  parse_ip(str) {
+    let chunks = str.split(":")
+    let port = chunks[chunks.length-1]
+    let host = chunks.slice(0, chunks.length-1).join(":")
+    return { host, port }
+  }
+}
+module.exports = Processor

package/kernel/shell.js

@@ -2,7 +2,7 @@ const { Terminal } = require('xterm-headless');
 const { SerializeAddon } = require("xterm-addon-serialize");
 const sanitize = require("sanitize-filename");
 const YAML = require('yaml')
-
+const kill = require('kill-sync')
 const fastq = require('fastq')
 const { v4: uuidv4 } = require('uuid');
 const os = require('os');
@@ -13,7 +13,6 @@ const pty = require('@homebridge/node-pty-prebuilt-multiarch')
 const path = require("path")
 const sudo = require("sudo-prompt-programfiles-x86");
 const unparse = require('yargs-unparser-custom-flag');
-const shellPath = require('shell-path');
 const Util = require('./util')
 const Environment = require('./environment')
 const home = os.homedir()
@@ -60,36 +59,7 @@ class Shell {
     }, 1)
 
   }
-  async start(params, ondata) {
-
-    /*
-      params := {
-        group: <group id>,
-        id: <shell id>,
-        path: <shell cwd (always absolute path)>,
-        env: <environment value key pairs>
-      }
-    */
-    this.cols = params.cols ? params.cols : 100;
-    this.rows = params.rows ? params.rows : 30;
-
-    this.vt = new Terminal({
-      allowProposedApi: true,
-      cols: this.cols,
-      rows: this.rows,
-    })
-    this.vts = new SerializeAddon()
-    this.vt.loadAddon(this.vts)
-
-    // 1. id
-    this.id = (params.id ? params.id : uuidv4())
-
-    // 2. group id
-    this.group = params.group
-
-    // 2. env
-    // default env
-
+  async init_env(params) {
     this.env = Object.assign({}, process.env)
     // If the user has set PYTHONPATH, unset it.
     if (this.env.PYTHONPATH) {
@@ -103,21 +73,29 @@ class Shell {
     if (this.env.CMAKE_GENERATOR) {
       delete this.env.CMAKE_GENERATOR
     }
+
+    //this.env.PNPM_CONFIG_PREFIX = this.kernel.path("bin/npm")
+    //this.env.pnpm_config_prefix = this.kernel.path("bin/npm")
+//    this.env.PNPM_HOME = this.kernel.path("bin/npm")
+//    this.env.pnpm_home = this.kernel.path("bin/npm")
+
+//    this.env.NPM_CONFIG_PREFIX = this.kernel.path("bin/npm")
+//    this.env.npm_config_prefix = this.kernel.path("bin/npm")
+
     
 //    if (this.env.CUDA_HOME) {
 //      delete this.env.CUDA_HOME
 //    }
     for(let key in this.env) {
       if (key.startsWith("CUDA")) {
-        console.log("Unset env key: " + key)
         delete this.env[key]
       }
       if (/.*(SSH|SSL).*/.test(key)) {
-        console.log("Unset env key: " + key)
         delete this.env[key]
       }
     }
 
+//    this.env.TCELL_MINIMIZE=1
     this.env.CMAKE_OBJECT_PATH_MAX = 1024
     this.env.PYTORCH_ENABLE_MPS_FALLBACK = 1
     this.env.TORCH_FORCE_NO_WEIGHTS_ONLY_LOAD = 1
@@ -152,7 +130,7 @@ class Shell {
     if (this.platform === 'win32') {
       // ignore 
     } else {
-      this.env[PATH_KEY]= shellPath.sync() || [
+      this.env[PATH_KEY]= this.kernel.shellpath || [
         './node_modules/.bin',
         '/.nodebrew/current/bin',
         '/usr/local/bin',
@@ -162,7 +140,6 @@ class Shell {
 
     if (this.platform === "linux") {
       let gxx = this.kernel.which('g++')
-      console.log({ gxx })
       if (gxx) {
         this.env.NVCC_PREPEND_FLAGS = `-ccbin ${gxx}`
       }
@@ -204,8 +181,6 @@ class Shell {
         }
       }
     }
-
-
     for(let key in this.env) {
       if (!/^[a-zA-Z_][a-zA-Z0-9_]*$/.test(key) && key !== "ProgramFiles(x86)") {
         delete this.env[key]
@@ -224,12 +199,68 @@ class Shell {
         delete this.env[key]
       }
     }
+  }
+  async start(params, ondata) {
+    this.ondata = ondata
+
+    /*
+      params := {
+        group: <group id>,
+        id: <shell id>,
+        path: <shell cwd (always absolute path)>,
+        env: <environment value key pairs>
+      }
+    */
+
+    this.kill_messages = params.kill
+
+    this.cols = 100
+    this.rows = 30
+
+    if (params.cols) {
+      this.cols = params.cols
+    }
+    if (params.rows) {
+      this.rows = params.rows
+    }
+    if (params.client && params.client.cols) {
+      this.cols = params.client.cols
+    }
+    if (params.client && params.client.rows) {
+      this.rows = params.client.rows
+    }
+
+    this.vt = new Terminal({
+      allowProposedApi: true,
+      cols: this.cols,
+      rows: this.rows,
+    })
+    this.vts = new SerializeAddon()
+    this.vt.loadAddon(this.vts)
+
+    // 1. id
+    this.id = (params.id ? params.id : uuidv4())
+
+    // 2. group id
+    this.group = params.group
+
+    // 2. env
+    // default env
+    await this.init_env(params)
+
+
+    this.ondata({ raw: `\r\n████\r\n██ Starting Shell ${this.id}\r\n` })
+
+
+    this.params = params
+
     // 3. path => path can be http, relative, absolute
     this.path = params.path
 
     // automatically add self to the shells registry
     this.kernel.shell.add(this)
 
+
     if (params.sudo) {
       let options = {
         name: "Pinokio",
@@ -275,20 +306,39 @@ class Shell {
       return response
     } else {
       let response = await this.request(params, async (stream) => {
-        if (stream.prompt) {
-          this.resolve()
-        } else {
-          if (ondata) ondata(stream)
-        }
+        if (ondata) ondata(stream)
+//        if (stream.prompt) {
+//          this.resolve()
+//        } else {
+//          if (ondata) ondata(stream)
+//        }
       })
       return response
     }
 
 //    return this.id
   }
+  emit2(message) {
+    let i = 0;
+    const interval = setInterval(() => {
+      if (i >= message.length) {
+        clearInterval(interval);
+        return;
+      }
+      let chunk = message.slice(i, i+1024)
+      this.ptyProcess.write(chunk)
+      i += 1024;
+    }, 10);
+  }
   emit(message) {
-    if (this.ptyProcess) {
-      this.ptyProcess.write(message)
+    if (message.length > 1024) {
+      this.emit2(message)
+    } else {
+      if (this.ptyProcess) {
+        if (this.input) {
+          this.ptyProcess.write(message)
+        }
+      }
     }
   }
   send(message, newline, cb) {
@@ -370,6 +420,9 @@ class Shell {
 
     // not connected => make a new connection => which means get a new prompt
     // if already connected => no need for a new prompt
+    if (params.input) {
+      this.input = params.input
+    }
     if (params.persistent) {
       this.persistent = params.persistent
     }
@@ -898,9 +951,11 @@ class Shell {
         this.resolve(buf)
       }
       this.resolve = undefined
+      this.ondata({ raw: `\r\n\r\n██ Detached from Shell ${this.id}\r\n\r\n` })
     }
   }
-  kill(message, force) {
+  kill(message, force, cb) {
+    console.log("KILL")
 
     this.done = true
     this.ready = false
@@ -924,12 +979,31 @@ class Shell {
     this.vt.dispose()
     this.queue.killAndDrain()
     if (this.ptyProcess) {
-      this.ptyProcess.kill()
-      this.ptyProcess = undefined
+      if (cb) {
+        kill(this.ptyProcess.pid, "SIGKILL", true)
+        this.ptyProcess.kill()
+        this.ptyProcess = undefined
+        // automatically remove the shell from this.kernel.shells
+        this.kernel.shell.rm(this.id)
+        this.ondata({ raw: `\r\n\r\n██ Terminated Shell ${this.id}\r\n████\r\n` })
+        this.ondata({ raw: "", type: "shell.kill" })
+        console.log("before cb 1")
+        cb()
+      } else {
+        kill(this.ptyProcess.pid, "SIGKILL", true)
+        this.ptyProcess.kill()
+        this.ptyProcess = undefined
+        // automatically remove the shell from this.kernel.shells
+        this.kernel.shell.rm(this.id)
+        this.ondata({ raw: `\r\n\r\n██ Terminated Shell ${this.id}\r\n████\r\n` })
+        this.ondata({ raw: "", type: "shell.kill" })
+      }
+    } else {
+      this.kernel.shell.rm(this.id)
+      this.ondata({ raw: `\r\n\r\n██ Terminated Shell ${this.id}\r\n████\r\n` })
+      this.ondata({ raw: "", type: "shell.kill" })
     }
 
-    // automatically remove the shell from this.kernel.shells
-    this.kernel.shell.rm(this.id)
 
   }
   log() {
@@ -1008,6 +1082,11 @@ ${cleaned}
 
   }
   stream(msg, callback) {
+    if (msg === "\u0007") {
+      // Ignore bell sound escape character because it slows down everything and makes redundant sound
+      callback()
+      return
+    }
     this.vt.write(msg, () => {
       let buf = this.vts.serialize()
       let cleaned = this.stripAnsi(buf)
@@ -1015,8 +1094,10 @@ ${cleaned}
         id: this.id,
         raw: msg,
         cleaned,
-        state: cleaned
+        state: cleaned,
+        shell_id: this.id
       }
+      this.state = cleaned
       if (this.cb) this.cb(response)
 
       // Decide whether to kill or continue
@@ -1031,15 +1112,15 @@ ${cleaned}
           // todo: may need to handle cases when the command returns immediately with no output (example: 'which brew' returns immediately with no text if brew doesn't exist)
           setTimeout(() => {
             if (cache === cleaned) {
-              if (this.persistent) {
-                if (this.cb) this.cb({
-                  //raw: cached_msg,
-                  //raw: msg,
-                  //raw: "",
-                  cleaned,
-                  state: cleaned,
-                  prompt: true
-                })
+              if (this.input || this.persistent) {
+//                if (this.cb) this.cb({
+//                  //raw: cached_msg,
+//                  //raw: msg,
+//                  //raw: "",
+//                  cleaned,
+//                  state: cleaned,
+//                  prompt: true
+//                })
                 callback()
               } else {
                 callback()

package/kernel/shells.js

@@ -35,14 +35,8 @@ class Shells {
     } else {
       m = "ls"
     }
-    let response = await sh.start({
-      message: m,
+    await sh.init_env({
       env: this.kernel.bin.envs({}),
-      conda: {
-        skip: true
-      }
-    }, async (stream) => {
-      process.stdout.write(stream.raw)
     })
 
     this.kernel.envs = sh.env
@@ -61,12 +55,14 @@ class Shells {
       let res
       for(let i=0; i<params.message.length; i++) {
         let message = params.message[i]
-        let params_dup = Object.assign({}, params) 
-        params_dup.message = message
-        res = await this._launch(params_dup, options, ondata)
-        // if there's an error, immediately return with the error
-        if (res.error) {
-          return res
+        if (message) {
+          let params_dup = Object.assign({}, params) 
+          params_dup.message = message
+          res = await this._launch(params_dup, options, ondata)
+          // if there's an error, immediately return with the error
+          if (res.error) {
+            return res
+          }
         }
       }
       return res
@@ -103,6 +99,7 @@ class Shells {
       break: false
     }]
     params.on = params.on.concat(defaultHandlers)
+
     let response = await sh.start(params, async (stream) => {
       /*
         {
@@ -150,7 +147,6 @@ class Shells {
       }
       ondata(stream)
     })
-
     /*
       {
         method: "shell.run",
@@ -350,18 +346,33 @@ class Shells {
   stop (request, message) {
     return this.kill(request, message)
   }
-  reset() {
+  reset(cb) {
     let info = this.shells.map((s) => {
       return { id: s.id, group: s.group, cmd: s.cmd }
     })
-    if (this.shells) {
+    if (this.shells && this.shells.length > 0) {
       let shells = []
       for(let i=0; i<this.shells.length; i++) {
         shells.push(this.shells[i])
       }
+      let count = 0
       for(let shell of shells) {
         console.log("[Kill Shell]", { id: shell.id, group: shell.group, cmd: shell.cmd })
-        shell.kill("", true)
+        if (cb) {
+          shell.kill("", true, () => {
+            count++
+            if (count >= shells.length) {
+              cb()
+            }
+          })
+        } else {
+          shell.kill("", true)
+        }
+      }
+    } else {
+      console.log("no shells running")
+      if (cb) {
+        cb()
       }
     }
   }
@@ -373,6 +384,7 @@ class Shells {
     *  - Kill by group ID
     *    request = { group } 
     */
+
     if (request.id) {
 
       let shells = []
@@ -451,6 +463,9 @@ class Shells {
         return shell.group === group
       })
       return found
+    } else if (request.filter) {
+      let found = this.shells.filter(request.filter)
+      return found
     }
   }
   get(id) {

package/kernel/store.js

@@ -0,0 +1,58 @@
+const os = require('os')
+const path = require('path')
+const fs = require('fs')
+class Store {
+  constructor() {
+    this.root = path.resolve(os.homedir(), ".pinokio")
+    if (!fs.existsSync(path.resolve(this.root))) {
+      fs.mkdirSync(path.resolve(this.root))
+    }
+  }
+  exists() {
+    let p =  path.resolve(this.root, "config.json")
+    return fs.existsSync(p)
+  }
+  clone(old) {
+    console.log("clone", old)
+    let p =  path.resolve(this.root, "config.json")
+    fs.writeFileSync(p, JSON.stringify(old, null, 2))
+  }
+  set(key, val) {
+    let o
+    try {
+      let str = fs.readFileSync(path.resolve(this.root, "config.json"), "utf8")
+      o = JSON.parse(str)
+    } catch(e) {
+      o = {}
+    }
+    o[key] = val
+    fs.writeFileSync(path.resolve(this.root, "config.json"), JSON.stringify(o, null, 2))
+  }
+  get(key) {
+    try {
+      let str = fs.readFileSync(path.resolve(this.root, "config.json"), "utf8")
+      let o = JSON.parse(str)
+      return o[key]
+    } catch (e) {
+      return null
+    }
+  }
+  delete(key) {
+    try {
+      let str = fs.readFileSync(path.resolve(this.root, "config.json"), "utf8")
+      let o = JSON.parse(str)
+      delete o[key]
+      fs.writeFileSync(path.resolve(this.root, "config.json"), JSON.stringify(o, null, 2))
+    } catch (e) {
+    }
+  }
+  store() {
+    try {
+      let str = fs.readFileSync(path.resolve(this.root, "config.json"), "utf8")
+      let o = JSON.parse(str)
+      return o
+    } catch (e) {
+    }
+  }
+}
+module.exports = Store

package/kernel/sysinfo.js

@@ -75,7 +75,6 @@ class Sysinfo {
     this.info.gpus = gpus
     this.info.gpu = gpu
     this.info.gpu_model = gpu_model
-    console.log("> this.info", this.info)
 
   }
 //  async time() {