pinokiod 3.9.0 → 3.9.3

package/kernel/procs.js

@@ -0,0 +1,116 @@
+const os = require('os')
+const path = require('path')
+const { fork, exec } = require('child_process');
+const platform = os.platform();
+class Procs {
+  constructor () {
+    console.log("Initializing procs")
+  }
+  getPortPidList(callback) {
+    const isWin = platform === 'win32';
+    const cmd = isWin ? 'netstat -ano -p tcp' : 'lsof -nP -iTCP -sTCP:LISTEN';
+    exec(cmd, (err, stdout) => {
+      if (err) return callback(err);
+
+      const results = [];
+      let pids = new Set()
+      const lines = stdout.trim().split('\n');
+
+      lines.forEach(line => {
+        if (isWin) {
+          // Skip headers
+          if (!line.startsWith('  TCP')) return;
+          const parts = line.trim().split(/\s+/);
+          const [ , localAddress, , state, pid ] = parts;
+          if (state !== 'LISTENING') return;
+          const chunks = localAddress.split(":")
+          const port = chunks.pop()
+          let ip = chunks.pop()
+          if (!ip || ip === "*") {
+            ip = "127.0.0.1:" + port
+          } else {
+            ip = localAddress
+          }
+          if (pids.has(pid+"/"+port)) return;
+          pids.add(pid+"/"+port)
+          results.push({ port, pid, ip });
+        } else {
+          if (!/LISTEN/.test(line)) return;
+          const parts = line.trim().split(/\s+/);
+          const pid = parts[1];
+
+          const match = line.match(/([^\s]+:\d+)\s/);
+          const localAddress = match?.[1];
+          const chunks = localAddress.split(":")
+          const port = chunks.pop()
+          let ip = chunks.pop()
+          if (!ip || ip === "*") {
+            ip = "127.0.0.1:" + port
+          } else {
+            ip = localAddress
+          }
+          //const portMatch = line.match(/:(\d+)\s/);
+          //const port = portMatch?.[1];
+          if (pids.has(pid+"/"+port)) return;
+          pids.add(pid+"/"+port)
+          if (pid && port) results.push({ port, pid, ip });
+        }
+      });
+
+      callback(null, results);
+    });
+  }
+  getPidToNameMap(callback) {
+    const isWin = platform === 'win32';
+    const cmd = isWin ? 'tasklist /fo csv /nh' : 'ps -Ao pid,comm';
+    exec(cmd, (err, stdout) => {
+      if (err) return callback(err);
+      const lines = stdout.trim().split('\n');
+      const map = {};
+      lines.forEach(line => {
+        if (isWin) {
+          const [name, pid] = line.split('","').map(s => s.replace(/(^"|"$)/g, ''));
+          map[pid] = name;
+        } else {
+          const [pid, ...nameParts] = line.trim().split(/\s+/);
+          map[pid] = nameParts.join(' ');
+        }
+      });
+      callback(null, map);
+    });
+  }
+  async refresh() {
+    let map = {}
+    let list = await new Promise((resolve, reject) => {
+      this.getPortPidList((err, portPidList) => {
+        if (err) {
+          reject(err)
+          return
+        }
+
+        this.getPidToNameMap((err, pidToName) => {
+          if (err) {
+            reject(err)
+            return
+          }
+
+          let list = portPidList.map(({ port, pid, ip }) => {
+            const fullname = pidToName[pid] || 'Unknown';
+            const name = fullname.split(path.sep).pop()
+            if (["caddy", "caddy.exe"].includes(name)) {
+              this.caddy_pid = pid
+              return null
+            } else {
+              map["" + pid] =  fullname
+              return { port, pid , name, fullname, ip }
+            }
+          }).filter((x) => { return x })
+          resolve(list)
+        });
+      });      
+    })
+    this.info = list
+    this.map = map
+  }
+}
+module.exports = Procs

package/kernel/prototype.js

@@ -0,0 +1,62 @@
+const fs = require('fs')
+const path = require('path')
+const { glob, sync, hasMagic } = require('glob-gitignore')
+const marked = require('marked')
+class Proto {
+  constructor(kernel) {
+    this.kernel = kernel
+  }
+  async init() {
+
+    // if ~/pinokio/prototype doesn't exist, clone
+    let exists = await this.kernel.exists("prototype")
+    if (!exists) {
+      console.log("prototype doesn't exist. cloning...")
+      await fs.promises.mkdir(this.kernel.path("prototype"), { recursive: true }).catch((e) => { })
+      await this.kernel.exec({
+        message: "git clone https://github.com/peanutcocktail/prototype system",
+        path: this.kernel.path("prototype")
+      }, (e) => {
+        process.stdout.write(e.raw)
+      })
+    }
+
+
+    let cwd = path.resolve(this.kernel.homedir, "prototype")
+    let prototype_paths = (await glob('**/pinokio.js', { cwd }))
+    let prototype_dir = path.resolve(this.kernel.homedir, "prototype")
+    this.items = []
+    this.kv = {}
+    for(let prototype_path of prototype_paths) {
+      let proto = path.dirname(prototype_path)
+      let pinokiojs = path.resolve(prototype_dir, prototype_path)
+      let config = await this.kernel.require(pinokiojs)
+      if (config && config.run) {
+        if (config.icon) {
+          config.icon = "/prototype/" + proto + "/" + config.icon
+        } else {
+          config.icon === "/pinokio-black.png"
+        }
+        let c = {
+          id: proto,
+          ...config
+        }
+        this.items.push(c)
+        this.kv[proto] = c
+      }
+    }
+  }
+  async readme(proto) {
+    console.log("proto.readme", proto)
+    let readme_path = path.resolve(this.kernel.homedir, "prototype", proto, "README.md")
+    let readme
+    try {
+      readme = await fs.promises.readFile(readme_path, "utf8")
+    } catch (e) {
+      readme = ""
+    }
+    console.log({ readme_path, readme })
+    return marked.parse(readme)
+  }
+}
+module.exports = Proto

package/kernel/router/common.js

@@ -0,0 +1,42 @@
+const Processor = require('./processor')
+class Common extends Processor {
+  constructor (router) {
+    super()
+    this.router = router
+  }
+  handle({ match, dial, host }) {
+    let handler = [{
+      "handler": "reverse_proxy",
+      "upstreams": [{ "dial": dial }],
+      "headers": {
+        "request": {
+          "set": {
+            "X-Forwarded-Proto": ["https"],
+            "X-Forwarded-Host": ["{http.request.host}"]
+          }
+        },
+        "response": {
+          "set": {
+            "Access-Control-Allow-Origin": ["*"],
+            "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+            "Access-Control-Allow-Headers": ["*"],
+            "Vary": ["Origin"]
+          }
+        }
+      }
+    }]
+
+    // if the dial port has been overridden by router.custom_routers, use that instead
+    let parsed_dial = this.parse_ip(dial)
+    let override_handler = this.router.custom_routers[String(parsed_dial.port)]
+    if (override_handler) {
+      handler = override_handler
+    }
+    this.router.config.apps.http.servers.main.routes.push({
+      "match": [{ "host": [match] }],
+      "handle": handler
+    })
+    this.router.add({ host, dial, match })
+  }
+}
+module.exports = Common

package/kernel/router/connector.js

@@ -0,0 +1,81 @@
+const Processor = require('./processor')
+class Connector extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+  }
+  handle ({ match, connector, dial, }) {
+    let connector_name = connector.host + ":" + connector.port
+    let listener = "0.0.0.0:PORT_PLACEHOLDER_" + connector_name
+    let listener_pointer = connector.host + ":PORT_PLACEHOLDER_" + connector_name
+    let port_placeholder = "PORT_PLACEHOLDER_" + connector_name
+
+    let handler = [{
+      "handler": "reverse_proxy",
+      "upstreams": [{ "dial": dial }],
+      "headers": {
+        "request": {
+          "set": {
+            "X-Forwarded-Proto": ["{http.request.header.X-Forwarded-Proto}"],
+            //"X-Forwarded-Proto": ["{http.request.scheme}"],
+            //"X-Forwarded-Proto": ["https"],
+            //"X-Forwarded-Host": ["{http.request.host}"]
+          }
+        },
+        "response": {
+          "set": {
+            "Access-Control-Allow-Origin": ["*"],
+            "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+            "Access-Control-Allow-Headers": ["*"],
+            "Vary": ["Origin"]
+          }
+        }
+      }
+    }]
+    // if the dial port has been overridden by router.custom_routers, use that instead
+    let parsed_dial = this.parse_ip(dial)
+    let override_handler = this.router.custom_routers[String(parsed_dial.port)]
+    if (override_handler) {
+      handler = override_handler
+    }
+    if(!this.router.config.apps.http.servers[connector_name]) {
+      this.router.config.apps.http.servers[connector_name] = {
+        "listen": [listener],
+        "routes": [{
+          "handle": handler
+        }]
+      }
+    }
+    this.router.config.apps.http.servers.main.routes.push({
+      "match": [{
+        "host": [match],
+      }],
+      "handle": [
+        {
+          "handler": "reverse_proxy",
+          "upstreams": [{ "dial": listener_pointer }],
+          "headers": {
+            "request": {
+              "set": {
+                //"X-Forwarded-Proto": ["https"],
+                //"Origin": "localhost",
+                "X-Forwarded-Proto": ["{http.request.scheme}"],
+                "X-Forwarded-Host": ["{http.request.host}"]
+              }
+            },
+            "response": {
+              "set": {
+                "Access-Control-Allow-Origin": ["*"],
+                "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                "Access-Control-Allow-Headers": ["*"],
+                "Vary": ["Origin"]
+              }
+            }
+          }
+        },
+      ]
+    })
+    this.router.add({ host: connector.host, dial: listener_pointer, match })
+  }
+}
+module.exports = Connector

package/kernel/router/index.js

@@ -0,0 +1,235 @@
+const axios = require('axios')
+const path = require('path')
+const { glob, sync, hasMagic } = require('glob-gitignore')
+const Util = require('../util')
+const LocalhostHomeRouter = require('./localhost_home_router')
+const LocalhostVariableRouter = require('./localhost_variable_router')
+const LocalhostPortRouter = require('./localhost_port_router')
+const PeerHomeRouter = require('./peer_home_router')
+const PeerVariableRouter = require('./peer_variable_router')
+const PeerPortRouter = require('./peer_port_router')
+const PeerPeerRouter = require('./peer_peer_router')
+const Environment = require("../environment")
+class Router {
+  constructor(kernel) {
+    this.kernel = kernel
+    this.localhost_home_router = new LocalhostHomeRouter(this)
+    this.localhost_variable_router = new LocalhostVariableRouter(this)
+    this.localhost_port_router = new LocalhostPortRouter(this)
+    this.peer_home_router = new PeerHomeRouter(this)
+    this.peer_variable_router = new PeerVariableRouter(this)
+    this.peer_port_router = new PeerPortRouter(this)
+    this.peer_peer_router = new PeerPeerRouter(this)
+    this.default_prefix = "pinokio"
+    this.default_suffix = "localhost"
+    this.default_match = this.default_prefix + "." + this.default_suffix
+    this.default_port = "42000"
+    this.default_host = "127.0.0.1"
+
+    this.info = {}
+    this.mapping = {}
+    this.port_mapping = {}   // 127.0.0.1 => 192.168,..
+    this.local_network_mapping = {}
+    this.custom_routers = {}
+  }
+  async init() {
+    // if ~/pinokio/network doesn't exist, clone
+    let exists = await this.kernel.exists("network/system")
+    if (!exists) {
+      console.log("network doesn't exist. cloning...")
+      await fs.promises.mkdir(this.kernel.path("network"), { recursive: true }).catch((e) => { })
+      await this.kernel.exec({
+        message: "git clone https://github.com/peanutcocktail/network system",
+        path: this.kernel.path("network")
+      }, (e) => {
+        process.stdout.write(e.raw)
+      })
+    }
+
+
+    let cwd = path.resolve(this.kernel.homedir, "network")
+    let router_paths = (await glob('**/*.json', { cwd }))
+    let router_dir = path.resolve(this.kernel.homedir, "network")
+
+
+    // create a custom_map that maps port to custom router declarations
+    /*
+      custom_routers = {
+        PORT1: <caddy handler>,
+        PORT2: <caddy handler>,
+        ...
+      }
+    */
+    this.custom_routers = {}
+    for(let router_path of router_paths) {
+      let router_abs_path = this.kernel.path("network", router_path)
+      let config = await this.kernel.require(router_abs_path)
+      for(let port in config) {
+        this.custom_routers[port] = config[port]
+      }
+    }
+
+    let env = await Environment.get(this.kernel.homedir)
+    this.active = false
+    if (env && env.PINOKIO_HTTPS_ACTIVE && (env.PINOKIO_HTTPS_ACTIVE==="1" || env.PINOKIO_HTTPS_ACTIVE.toLowerCase()==="true")) {
+      this.active = true
+    }
+
+
+  }
+  async add({ host, match, dial }) {
+    if (!this._mapping[host]) {
+      this._mapping[host] = {}
+    }
+    if (!this._mapping[host][dial]) {
+      this._mapping[host][dial] = new Set()
+    }
+    this._mapping[host][dial].add(match)
+  }
+  _info() {
+    let mapping = {}
+    for(let host in this.mapping) {
+      let internal_maps = this.mapping[host]
+      if (!mapping[host]) {
+        mapping[host] = {}
+      }
+      for(let url in internal_maps) {
+        mapping[host][url] = Array.from(internal_maps[url])
+      }
+    }
+    let host_mapping = mapping[this.kernel.peer.host]
+    if (host_mapping) {
+      for(let key in host_mapping) {
+        for(let cache_key in this.port_cache) {
+          if (key.endsWith(cache_key)) {
+            let transformed = key.replace(":" + cache_key, '')
+            let port = this.port_cache[cache_key]
+            host_mapping[transformed + ":" + port] = host_mapping[key]
+            delete host_mapping[key]
+          }
+        }
+      }
+      mapping[this.kernel.peer.host] = host_mapping
+    }
+    return mapping
+  }
+  /*
+    returns only the published peer info
+    {
+      <peer1 host>: {
+        <dial1>: [<match1>, <match2>, ..],
+        <dial2>: [<match1>, <match2>, ..],
+      },
+      <peer2 host>: {
+        <dial1>: [<match1>, <match2>, ..],
+        <dial2>: [<match1>, <match2>, ..],
+      },
+    }
+  */
+  published() {
+    let pub = {}
+    if (this.info) {
+      let routes = this.info[this.kernel.peer.host]
+      for(let dial in routes) {
+        let matches = routes[dial]
+        pub[dial] = matches
+      }
+    }
+    return pub
+  }
+  async fill() {
+    const jsonString = JSON.stringify(this.config)
+    // find all PORT_PLACEHOLDER patterns
+    const matches = jsonString.match(/:PORT_PLACEHOLDER_[0-9.:]+/g)
+
+    // iterate through them and see if the port already exists
+    let count = matches ? new Set(matches).size : 0
+    let new_config
+    if (count > 0) {
+      let ports = await this.kernel.ports(count)
+      let response = Util.fill_object(this.config, /PORT_PLACEHOLDER_.+/, ports, this.port_cache)
+      this.config = response.result
+      let replaced_map = response.replaced_map
+      for(let key in replaced_map) {
+        // local_share_ip :- 192.168....
+        let local_share_key = key.replace("PORT_PLACEHOLDER_", "")
+        let chunks = local_share_key.split(":")
+        let original_port = chunks[chunks.length-1]
+        let local_share_ip = chunks.slice(0, chunks.length-1).join(":")
+        let local_share_port = replaced_map[key]
+        if (!this.local_network_mapping[local_share_ip]) {
+          this.local_network_mapping[local_share_ip] = {}
+        }
+        this.local_network_mapping[local_share_ip][original_port] = local_share_port
+        //let original_port = key.replace("PORT_PLACEHOLDER_", "")
+        this.port_mapping[original_port] = replaced_map[key]
+      }
+      this.port_cache = replaced_map
+    }
+  }
+
+
+  // set local config
+  async local() {
+    this._mapping = {}
+    this.localhost_home_router.handle()
+    this.localhost_variable_router.handle(this.kernel.memory.local)
+    for(let proc of this.kernel.processes.info) {
+      this.localhost_port_router.handle(proc)
+    }
+    if (this.kernel.peer.active) {
+      for(let host in this.kernel.peer.info) {
+        let peer = this.kernel.peer.info[host]
+        if (peer.host === this.kernel.peer.host) {
+          this.peer_home_router.handle(peer)
+          this.peer_variable_router.handle(peer)
+          this.peer_port_router.handle(peer)
+        }
+      }
+      await this.fill()
+    }
+    this.mapping = this._mapping
+  }
+
+  // set remote config
+  async remote() {
+    for(let host in this.kernel.peer.info) {
+      let peer = this.kernel.peer.info[host]
+      if (peer.host !== this.kernel.peer.host) {
+        this.peer_peer_router.handle(peer)
+      }
+    }
+    //await this.fill()
+    this.mapping = this._mapping
+    this.info = this._info()
+  }
+
+  // update caddy config
+  async update() {
+    if (JSON.stringify(this.config) === JSON.stringify(this.old_config)) {
+//      console.log("config hasn't updated")
+    } else {
+      console.log("caddy config has updated", this.config)
+      try {
+        let response = await axios.post('http://127.0.0.1:2019/load', this.config, {
+          headers: { 'Content-Type': 'application/json' }
+        })
+//        console.log("Caddy Response", { response })
+        this.old_config = this.config
+      } catch (e) {
+        console.log("EEEEEEEEEEEEE", e)
+      }
+    }
+  }
+  async check() {
+    try {
+      let res = await axios.get(`http://localhost:2019/config/`, {
+        timeout: 2000
+      })
+      return res.data
+    } catch (e) {
+      return null
+    }
+  }
+}
+module.exports = Router

package/kernel/router/localhost_home_router.js

@@ -0,0 +1,84 @@
+class LocalhostHomeRouter {
+  constructor(router) {
+    this.router = router
+  }
+  handle () {
+    this.router.add({ host: this.router.kernel.peer.host, dial: this.router.default_host + ":" + this.router.default_port, match: this.router.default_match })
+    this.router.config = {
+      "apps": {
+        "http": {
+          "servers": {
+            "main": {
+              "listen": [":443"],
+              "routes": [
+                {
+                  "match": [{ "method": ["OPTIONS"] }],
+                  "handle": [
+                    {
+                      "handler": "headers",
+                      "response": {
+                        "set": {
+                          "Access-Control-Allow-Origin": ["*"],
+                          "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                          "Access-Control-Allow-Headers": ["*"],
+                          "Vary": ["Origin"],
+                        }
+                      }
+                    },
+                    {
+                      "handler": "static_response",
+                      "status_code": 204
+                    }
+                  ]
+                },
+                {
+                  "match": [{ "host": [ this.router.default_match ] }],
+                  "handle": [
+                    {
+                      "handler": "reverse_proxy",
+                      "upstreams": [{ "dial": this.router.default_host + ":" + this.router.default_port }],
+                      "headers": {
+                        "request": {
+                          "set": {
+                            "X-Forwarded-Proto": ["https"],
+                            "X-Forwarded-Host": ["{http.request.host}"]
+                          }
+                        },
+                        "response": {
+                          "set": {
+                            "Access-Control-Allow-Origin": ["*"],
+                            "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                            "Access-Control-Allow-Headers": ["*"],
+                            "Vary": ["Origin"]
+                          }
+                        }
+                      }
+                    },
+                  ]
+                },
+              ]
+            }
+          }
+        }
+      },
+      "logging": {
+        "logs": {
+          "default": {
+            "writer": {
+              "output": "file",
+              "filename": this.router.kernel.path("logs/caddy.log"),
+              "roll": true,
+              "roll_size_mb": 1,
+              "roll_keep": 1,
+              "roll_keep_days": 1,
+              "roll_gzip": false,
+              "roll_local_time": true
+            },
+            "level": "INFO"
+          }
+        }
+      }
+    }
+  }
+}
+module.exports = LocalhostHomeRouter

package/kernel/router/localhost_port_router.js

@@ -0,0 +1,15 @@
+const Common = require('./common')
+class LocalhostPortRouter {
+  constructor(router) {
+    this.router = router
+    this.common = new Common(router)
+  }
+  handle (proc) {
+    this.common.handle({
+      match: `${proc.port}.localhost`,
+      dial: proc.ip,
+      host: this.router.kernel.peer.host,
+    })
+  }
+}
+module.exports = LocalhostPortRouter

package/kernel/router/localhost_variable_router.js

@@ -0,0 +1,37 @@
+const path = require('path')
+const Common = require('./common')
+const Processor = require('./processor')
+class LocalhostVariableRouter extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+    this.common = new Common(router)
+  }
+  handle (local) {
+    for(let script_path in local) {
+      let local_variables_for_script = local[script_path]
+      for(let key in local_variables_for_script) {
+        let val = local_variables_for_script[key]
+        if (val.startsWith("http")) {
+          let dial = val.replace(/https?:\/\//, '')
+          let name = this.api_name(this.router.kernel.platform, this.router.kernel.homedir, script_path)
+          if (this.has_port(dial)) {
+            if (key === "url") {
+              this.common.handle({
+                match: `${name}.localhost`.toLowerCase(),
+                dial,
+                host: this.router.kernel.peer.host,
+              })
+            }
+            this.common.handle({
+              match: `${key}.${name}.localhost`.toLowerCase(),
+              dial,
+              host: this.router.kernel.peer.host,
+            })
+          }
+        }
+      }
+    }
+  }
+}
+module.exports = LocalhostVariableRouter

package/kernel/router/peer_home_router.js

@@ -0,0 +1,15 @@
+const Common = require('./common')
+class PeerHomeRouter {
+  constructor(router) {
+    this.router = router
+    this.common = new Common(router)
+  }
+  handle(peer) {
+    this.common.handle({
+      match: `${this.router.default_prefix}.${peer.name}.${this.router.default_suffix}`,
+      dial: `${peer.host}:${this.router.default_port}`,
+      host: peer.host
+    })
+  }
+}
+module.exports = PeerHomeRouter