pinokiod 3.9.0 → 3.9.3

package/kernel/index.js

@@ -7,6 +7,7 @@ const fastq = require('fastq')
 const fetch = require('cross-fetch');
 const waitOn = require('wait-on');
 const system = require('systeminformation');
+const shellPath = require('shell-path');
 const Sysinfo = require('./sysinfo')
 const portfinder = require('portfinder-cp');
 const { execSync } = require('child_process')
@@ -19,6 +20,7 @@ const Template = require('jimini')
 const which = require('which')
 const Shells = require("./shells")
 const Key = require("./key")
+const KV = require('./kv')
 const Script = require('./script')
 const Environment = require("./environment")
 const Util = require('./util')
@@ -26,6 +28,14 @@ const Config = require("./pinokio.json")
 const Info = require('./info')
 const Pipe = require("../pipe")
 const Cloudflare = require('./api/cloudflare')
+const Store = require('./store')
+const Proto = require('./prototype')
+const Router = require("./router")
+const Procs = require('./procs')
+const Peer = require('./peer')
+const Connect = require('./connect')
+//const kill = require('./tree-kill');
+const kill = require('kill-sync')
 const VARS = {
   pip: {
     install: {
@@ -36,13 +46,31 @@ const VARS = {
 
 //const memwatch = require('@airbnb/node-memwatch');
 class Kernel {
-  //schema = ">=1.0.0"
-  //schema = "<=2.1.0"
-  //schema = "<=3.2.0"
-  schema = "<=3.7.0"
+  schema = "<=4.0.0"
   constructor(store) {
     this.fetch = fetch
-    this.store = store
+
+    // STORE
+    // Previously custom STORE could be passed in. This was to support both electron and server based initialization
+    // But this creates complexity because the store location is not fixed
+    // Migrating it to ~/.pinokio/config.json
+    //
+    // 1. does ~/.pinokio exist?
+    //  - load from that file
+    // 2. does ~/.pinokio NOT exist?
+    //  - save the passed in options to the `~/.pinokio`
+    //  - load the config
+
+    this.store = new Store()
+    let exists = this.store.exists()
+    console.log("Store exists?", exists)
+    if (!exists) {
+      console.log("doesn't exist")
+      // clone the store to the new store
+      this.store.clone(store.store)
+      // load from the store (this will be the last time this is used, since the next time it loads, it will load from the new store)
+    }
+
     this.arch = os.arch()
     this.os = os
     this.platform = os.platform()
@@ -50,9 +78,9 @@ class Kernel {
     this.jsdom = jsdom
     this.exposed = {}
     this.envs = {}
-    this.info = new Info(this)
-    this.pipe = new Pipe(this)
-    this.cloudflare = new Cloudflare()
+    this.shellpath = shellPath.sync()
+
+
   }
   /*
     kernel.env() => return the system environment
@@ -147,6 +175,15 @@ class Kernel {
     let id = this.api.filePath(uri, cwd)
     return this.api.running[id]
   }
+  start_port () {
+    if (this.router.port_mapping && Object.keys(this.router.port_mapping).length > 0) {
+      let max_caddy_port = Math.max(...Object.values(this.router.port_mapping))
+      let start_port = Math.max(42003, max_caddy_port + 1)
+      return start_port
+    } else {
+      return 42003
+    }
+  }
   port(port) {
     // 1. if port is passed, check port
     // 2. if no port is passed, get the next available port
@@ -158,9 +195,21 @@ class Kernel {
     if (port) {
       return portfinder.isAvailablePromise({ host: "0.0.0.0", port })
     } else {
-      return portfinder.getPortPromise({ port: 42003 })
+      return portfinder.getPortPromise({ port: this.start_port() })
     }
   }
+  ports(count) {
+    // get "count" number of available ports
+    return new Promise((resolve, reject) => {
+      portfinder.getPorts(count, { port: this.start_port() }, (err, ports) => {
+        if (err) {
+          reject(err)
+        } else {
+          resolve(ports)
+        }
+      })
+    })
+  }
   path(...args) {
     return path.resolve(this.homedir, ...args)
   }
@@ -185,6 +234,56 @@ class Kernel {
   log(data, group, info) {
     this.log_queue.push({ data, group, info })
   }
+  async refresh(notify_peers) {
+    let env = await Environment.get(this.homedir)
+    let peer_active = false
+    // if PINOKIO_NETWORK_SHARE is 0 or false, turn it off
+    if (env && env.PINOKIO_NETWORK_ACTIVE && (env.PINOKIO_NETWORK_ACTIVE==="1" || env.PINOKIO_NETWORK_ACTIVE.toLowerCase()==="true")) {
+      peer_active = true
+    }
+    let https_active = false
+    if (env && env.PINOKIO_HTTPS_ACTIVE && (env.PINOKIO_HTTPS_ACTIVE==="1" || env.PINOKIO_HTTPS_ACTIVE.toLowerCase()==="true")) {
+      https_active = true
+    }
+//    console.log("kernel.refresh", { active, notify_peers })
+
+    let caddy_installed = await this.bin.check_installed({ name: "caddy" })
+
+    // 1. https
+    // 2. https + local share
+//    console.log("caddy installed?", caddy_installed)
+    if (caddy_installed && https_active) {
+
+      // 1. get the process list
+      await this.processes.refresh()
+
+      // 2. refresh peer info to reflect the proc info
+      await this.peer.refresh()
+
+      // 3. load custom routers from ~/pinokio/network
+      await this.router.init()
+
+      // 4. set current local host router info
+      await this.router.local()
+
+      // 5. refresh peer info to reflect the updated router info
+      await this.peer.refresh()
+
+      // 6. tell peers to refresh
+      if (notify_peers) {
+        await this.peer.notify_peers()
+      }
+
+      // 7. update remote router
+      await this.router.remote()
+
+      // 8. update caddy config
+      await this.router.update()
+
+      // 9. announce self to the peer network
+      this.peer.announce()
+    }
+  }
   async clearLog(group) {
     let relativePath = path.relative(this.homedir, group)
     for(let type of ["info", "buf", "cleaned"]) {
@@ -236,12 +335,9 @@ class Kernel {
   }
   async getInfo(refresh){
     let info = this.sysinfo
-    console.time(">>>>> updateSysinfo")
-    console.log("refresh", refresh)
     if (refresh) {
       await this.update_sysinfo()
     }
-    console.timeEnd(">>>>> updateSysinfo")
     let i = {
       version: this.version,
       platform: this.platform,
@@ -313,8 +409,20 @@ class Kernel {
     let folders = files.filter((f) => { return f.isDirectory() }).map((x) => { return x.name })
     let meta = {}
     for(let folder of folders) {
-      let p = path.resolve(this.api.userdir, folder, "pinokio.js")
-      let pinokio = (await this.loader.load(p)).resolved
+
+      let pinokiojson = this.path("api", folder, "pinokio.json")
+      let pinokiometajson = this.path("api", folder, "pinokio_meta.json")
+      let pinokiojs = this.path("api", folder, "pinokio.js")
+      let exists1 = await this.exists(pinokiojson)
+      let exists2 = await this.exists(pinokiometajson)
+      let exists3 = await this.exists(pinokiojs)
+      if (!exists1 && !exists2 && !exists3) {
+        await fs.promises.writeFile(pinokiojson, JSON.stringify({
+          title: "No title",
+          description: ""
+        }))
+      }
+      let pinokio = await this.api.meta(folder)
       if (pinokio) {
         i.api.push({
           path: folder,
@@ -390,13 +498,28 @@ class Kernel {
 //      }
     }
   }
+  kill() {
+    process.kill(process.pid, "SIGTERM")
+  }
   async init(options) {
+
     let home = this.store.get("home")
 
     // reset shells if they exist
     if (this.shell) {
       this.shell.reset()
     }
+    if (this.peer) {
+      this.peer.stop()
+    }
+
+    this.info = new Info(this)
+    this.pipe = new Pipe(this)
+    this.proto = new Proto(this)
+    this.processes = new Procs()
+    this.kv = new KV(this)
+    this.cloudflare = new Cloudflare()
+    this.peer = new Peer()
 
     this.homedir = home
 
@@ -443,6 +566,8 @@ class Kernel {
     this.api = new Api(this)
     this.python = new Python(this)
     this.shell = new Shells(this)
+    this.router = new Router(this)
+    this.connect = new Connect(this)
     this.system = system
     this.keys = {}
     this.memory = {
@@ -462,7 +587,6 @@ class Kernel {
 
         // 0. create homedir
         let home_exists = await this.exists(this.homedir)
-        console.log({ home_exists, homedir: this.homedir })
         if (!home_exists) {
           // home didn't exist.
           // 1. mkdir
@@ -487,7 +611,6 @@ class Kernel {
           NO_PROXY: (this.store.get("NO_PROXY") || "")
         }
         let fullpath = path.resolve(this.homedir, "ENVIRONMENT")
-        console.log("> update env", { fullpath, updated })
         await Util.update_env(fullpath, updated)
 
         // 2. mkdir all the folders if not already created
@@ -510,9 +633,9 @@ class Kernel {
       //await this.bin.init()
       let ts = Date.now()
       this.bin.init().then(() => {
-        console.log("bin init finished")
         if (this.homedir) {
           this.shell.init().then(async () => {
+//            this.bin.check()
             if (this.envs) {
               this.template.update({
                 env: this.envs,
@@ -542,16 +665,12 @@ class Kernel {
                 let interval = setInterval(async () => {
                   if (this.i) {
                     for(let api of this.i.api) {
-                      console.log({ api })
                       let env_path = path.resolve(this.api.userdir, api.path)
                       let e = await Environment.get(env_path)
-                      console.log(e)
                       if (e.PINOKIO_SCRIPT_AUTOLAUNCH && e.PINOKIO_SCRIPT_AUTOLAUNCH.trim().length > 0) {
                         let autolaunch_path = path.resolve(env_path, e.PINOKIO_SCRIPT_AUTOLAUNCH)
                         let exists = await this.exists(autolaunch_path)
-                        console.log("ATTEMPTING AUTOLAUNCH", autolaunch_path)
                         if (exists) {
-                          console.log("SCRIPT EXISTS. Starting...", autolaunch_path)
                           this.api.process({
                             uri: autolaunch_path,
                             input: {}
@@ -566,7 +685,6 @@ class Kernel {
                         }
                       }
                     }
-                    console.log("clear Interval")
                     clearInterval(interval)
                     setTimeout(() => {
                       this.launch_complete = true
@@ -580,14 +698,11 @@ class Kernel {
         }
       })
       let ts2 = Date.now()
-      console.time(">>>> 6 api.init")
       await this.api.init()
-      console.timeEnd(">>>> 6 api.init")
 
       //await this.shell.init()
 
       if (this.homedir) {
-        console.time(">>>> 7 template.init")
         await this.template.init({
           kernel: this,
           system,
@@ -597,25 +712,30 @@ class Kernel {
             return this.api.get_proxy_url("/proxy", port)
           },
         })
-        console.timeEnd(">>>> 7 template.init")
         this.sys = new Sysinfo()
-        console.time("sys init")
-        console.time(">>>> 8 sys.init")
         await this.sys.init(this)
-        console.timeEnd(">>>> 8 sys.init")
-        console.timeEnd("sys init")
         let info = this.sys.info
 
         this.sysinfo = info
 
-        console.time(">>>> 9 getInfo")
         await this.getInfo()
-        console.timeEnd(">>>> 9 getInfo")
 
         await fs.promises.mkdir(this.path("logs"), { recursive: true }).catch((e) => { })
         await fs.promises.writeFile(this.path("logs/system.json"), JSON.stringify(this.i, null, 2))
         let pwpath = this.bin.path("playwright/node_modules/playwright")
         this.playwright = (await this.loader.load(pwpath)).resolved
+
+
+        await this.proto.init()
+
+        // refresh every 5 second
+        if (this.refresh_interval) {
+          clearInterval(this.refresh_interval)
+        }
+        this.refresh_interval = setInterval(() => {
+          this.refresh()    
+        }, 5000)
+
       }
 
     } catch (e) {

package/kernel/info.js

@@ -40,12 +40,14 @@ class Info {
   running(...args) {
     let cwd = path.dirname(this.caller())
     let resolved_path = path.resolve(cwd, ...args)
-    return this.kernel.api.running[resolved_path]
+    let running = this.kernel.api.running[resolved_path]
+    return running ? running : false
   }
   exists(...args) {
     let cwd = path.dirname(this.caller())
     let resolved_path = path.resolve(cwd, ...args)
-    return fs.existsSync(resolved_path)
+    let exists = fs.existsSync(resolved_path)
+    return exists ? exists: false
   }
   local(...args) {
     let cwd = path.dirname(this.caller())

package/kernel/kv.js

@@ -0,0 +1,38 @@
+class KV {
+  constructor(kernel) {
+    this.kernel = kernel
+  }
+  async set(host, val, index) {
+    if (this.kernel.homedir) {
+      let filePath = this.kernel.path("key.json")
+      let json = await this.kernel.load(filePath)
+      if (!json) json = {}
+      // if the array at host path doesn't exist, create an empty array
+      if (!json[host]) {
+        json[host] = []
+      }
+      // if index doesn't exist, set index 0
+      if (!index) {
+        index = 0
+      }
+      json[host][index] = val
+      await fs.promises.writeFile(filePath, JSON.stringify(json, null, 2), 'utf8')
+    }
+  }
+  async get(host, index) {
+    if (this.kernel.homedir) {
+      let filePath = this.kernel.path("key.json")
+      console.log({ host, index, filePath })
+      let json = await this.kernel.load(filePath)
+      if (!json) {
+        return null
+      }
+      console.log({ json })
+      if (json[host]) {
+        return json[host][index]
+      }
+    }
+    return null
+  }
+}
+module.exports = KV

package/kernel/loader.js

@@ -11,6 +11,11 @@ class Loader {
     let extension = path.extname(_path)
     clearModule(_path)
 
+    let exists = await this.exists(_path)
+    if (!exists) {
+      return { resolved: null, dirname: path.dirname(_path), }
+    }
+
     if (/\.json$/i.test(_path)) {
       resolved = await this.requireJSON(_path)
       dirname = path.dirname(_path)
@@ -32,23 +37,25 @@ class Loader {
           resolved = null
         }
       } catch (e) {
-        console.log(e)
         resolved = null
       }
     }
+    if (!resolved) {
+      console.log(`[did not load] ${_path}`)
+    }
     return { resolved, extension, dirname }
   }
   async requireJSON(filepath) {
     let config
     try { config = require(filepath) } catch (e) {
-      console.log("load error", filepath, e)
+      console.log("> load", e, filepath)
     }
     return config
   }
   async requireJS(filepath) {
     let config
     try { config = require(filepath) } catch (e) {
-      console.log("load error", filepath, e)
+      console.log("> load", e, filepath)
     }
     try {
       // if the required module is a class, return the instantiated object
@@ -79,5 +86,8 @@ class Loader {
     })
     return result
   }
+  exists(abspath) {
+    return new Promise(r=>fs.access(abspath, fs.constants.F_OK, e => r(!e)))
+  }
 }
 module.exports = Loader

package/kernel/peer.js

@@ -0,0 +1,161 @@
+const dgram = require('dgram');
+const axios = require('axios');
+const os = require('os')
+const Environment = require("./environment")
+class PeerDiscovery {
+  constructor(port = 41234, message = 'ping', interval = 1000) {
+    this.port = port;
+    this.message = Buffer.from(message);
+    this.interval = interval;
+    this.peers = new Set();
+    this.host = this._getLocalIPAddress()
+    this.default_port = 42000
+    this.peers.add(this.host)
+//    this.start();
+  }
+  stop() {
+    if (this.socket) {
+      console.log("peer stop")
+      clearInterval(this.interval_handle)
+      this.socket.close()
+    }
+  }
+  announce() {
+    if (this.socket) {
+      this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
+    }
+  }
+  async start(kernel) {
+    let env = await Environment.get(kernel.homedir)
+
+    // by default expose to the local network
+    this.active = false
+    // if PINOKIO_NETWORK_SHARE is 0 or false, turn it off
+    if (env && env.PINOKIO_NETWORK_ACTIVE && (env.PINOKIO_NETWORK_ACTIVE==="1" || env.PINOKIO_NETWORK_ACTIVE.toLowerCase()==="true")) {
+      this.active = true
+    }
+
+    this.name = os.userInfo().username
+    if (env && env.PINOKIO_NETWORK_NAME && env.PINOKIO_NETWORK_NAME.length > 0) {
+      this.name = env.PINOKIO_NETWORK_NAME
+    }
+
+    if (this.active) {
+      // Listen for incoming pings
+      this.socket = dgram.createSocket('udp4');
+      this.socket.on('message', (msg, rinfo) => {
+        const ip = rinfo.address;
+        if (msg.toString() === this.message.toString() && this._isLocalLAN(ip)) {
+          if (!this.peers.has(ip)) {
+            console.log(`Discovered peer: ${ip}`);
+            this.peers.add(ip);
+            this.refresh()
+          }
+        }
+      });
+      this.socket.on('error', (err) => {
+        console.error('UDP error', err);
+      });
+
+      // Enable broadcast
+      console.log("binding socket")
+      this.socket.bind({ address: "0.0.0.0", port: this.port }, () => {
+        console.log("socket bound")
+//        this.socket.setMulticastLoopback(true);
+
+        this.socket.setBroadcast(true);
+
+        this.announce()
+
+//        // Send broadcast pings every interval
+//        this.interval_handle = setInterval(() => {
+//          //this.socket.send(this.message, 0, this.message.length, this.port, '255.255.255.255');
+//          this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
+//        }, this.interval);
+      });
+    }
+  }
+  async _refresh(host) {
+    try {
+      let res = await axios.get(`http://${host}:${this.default_port}/pinokio/peer`, {
+        timeout: 2000
+      })
+      return res.data
+    } catch (e) {
+      return null
+    }
+  }
+  async _broadcast(host) {
+    try {
+      let res = await axios.post(`http://${host}:${this.default_port}/pinokio/peer/refresh`, {
+        timeout: 2000
+      })
+      return res.data
+    } catch (e) {
+      return null
+    }
+  }
+  // notify peers to refresh my info
+  async notify_peers() {
+    if (this.active) {
+      let res = await Promise.all(Array.from(this.peers).map((host) => {
+        return this._broadcast(host)
+      }))
+      return res
+    }
+  }
+  // refresh peer info
+  async refresh(peers) {
+//    if (this.active) {
+      this.refreshing = true
+      let refresh_peers
+      if (peers) {
+        refresh_peers = peers
+      } else {
+        this.info = {}
+        refresh_peers = Array.from(this.peers)
+      }
+      let peer_info = await Promise.all(refresh_peers.map((host) => {
+        return this._refresh(host)
+      }))
+      for(let i=0; i<peer_info.length; i++) {
+        let peer = peer_info[i]
+        if (peer) {
+          this.info[peer.host] = {
+            host: peer.host,
+            ...peer
+          }
+        }
+      }
+      this.refreshing = false
+      return this.info
+//    }
+  }
+  _isLocalLAN(ip) {
+    return ip.startsWith('192.168.') || ip.startsWith('10.') || (ip.startsWith('172.') && is172Private(ip));
+  }
+  _getLocalIPAddress() {
+    const interfaces = os.networkInterfaces();
+    for (const ifaceList of Object.values(interfaces)) {
+      for (const iface of ifaceList) {
+        if (iface.family === 'IPv4' && !iface.internal) {
+          const ip = iface.address;
+          if (
+            ip.startsWith('10.') ||
+            ip.startsWith('192.168.') ||
+            (ip.startsWith('172.') && is172Private(ip))
+          ) {
+            return ip;
+          }
+        }
+      }
+    }
+    return null;
+    function is172Private(ip) {
+      const secondOctet = parseInt(ip.split('.')[1], 10);
+      return secondOctet >= 16 && secondOctet <= 31;
+    }
+  }
+}
+
+module.exports = PeerDiscovery;