pinokiod 3.8.44 → 3.8.45

package/kernel/router/index.js

@@ -0,0 +1,137 @@
+const axios = require('axios')
+const path = require('path')
+const Util = require('../util')
+const LocalhostHomeRouter = require('./localhost_home_router')
+const LocalhostVariableRouter = require('./localhost_variable_router')
+const LocalhostPortRouter = require('./localhost_port_router')
+const PeerHomeRouter = require('./peer_home_router')
+const PeerVariableRouter = require('./peer_variable_router')
+const PeerPortRouter = require('./peer_port_router')
+class Router {
+  constructor(kernel) {
+    this.kernel = kernel
+    this.localhost_home_router = new LocalhostHomeRouter(this)
+    this.localhost_variable_router = new LocalhostVariableRouter(this)
+    this.localhost_port_router = new LocalhostPortRouter(this)
+    this.peer_home_router = new PeerHomeRouter(this)
+    this.peer_variable_router = new PeerVariableRouter(this)
+    this.peer_port_router = new PeerPortRouter(this)
+    this.default_prefix = "pinokio"
+    this.default_suffix = "localhost"
+    this.default_match = this.default_prefix + "." + this.default_suffix
+    this.default_port = "42000"
+    this.default_host = "127.0.0.1"
+    this.config
+  }
+  async add({ host, match, dial }) {
+    if (!this._mapping[host]) {
+      this._mapping[host] = {}
+    }
+    if (!this._mapping[host][dial]) {
+      this._mapping[host][dial] = new Set()
+    }
+    this._mapping[host][dial].add(match)
+  }
+  info() {
+    let mapping = {}
+    for(let host in this._mapping) {
+      let internal_maps = this._mapping[host]
+      if (!mapping[host]) {
+        mapping[host] = {}
+      }
+      for(let url in internal_maps) {
+        mapping[host][url] = Array.from(internal_maps[url])
+      }
+    }
+    return mapping
+  }
+  async fill() {
+    const jsonString = JSON.stringify(this.config)
+    // find all PORT_PLACEHOLDER patterns
+    const matches = jsonString.match(/:PORT_PLACEHOLDER_[0-9.:]+/g)
+    console.log("matches", matches)
+
+    // iterate through them and see if the port already exists
+    let count = 0
+    if (matches) {
+      for(let match of matches) {
+        console.log("testing", match)
+        const port_match = match.match(/[0-9.]+:([0-9]+)/);
+        let port = port_match ? port_match[1] : null
+        console.log({ port, port_match })
+        if (port && !this.port_mapping[port]) {
+          count++
+        }
+      }
+    }
+
+    console.log("count", count)
+ 
+    if (count > 0) {
+      let ports = await this.kernel.ports(count)
+      console.log({ count, ports })
+      console.log("PORT_CACHE", this.port_cache)
+      let response = Util.fill_object(this.config, /PORT_PLACEHOLDER_.+/, ports, this.port_cache)
+      this.config = response.result
+      let replaced_map = response.replaced_map
+      for(let key in replaced_map) {
+        // local_share_ip :- 192.168....
+        let local_share_key = key.replace("PORT_PLACEHOLDER_", "")
+        let chunks = local_share_key.split(":")
+        let local_share_ip = chunks[0]
+        let original_port = chunks[1]
+        let local_share_port = replaced_map[key]
+        console.log({ local_share_ip, local_share_port })
+        if (!this.local_network_mapping[local_share_ip]) {
+          this.local_network_mapping[local_share_ip] = {}
+        }
+        this.local_network_mapping[local_share_ip][original_port] = local_share_port
+
+
+        //let original_port = key.replace("PORT_PLACEHOLDER_", "")
+        this.port_mapping[original_port] = replaced_map[key]
+      }
+      this.port_cache = replaced_map
+    }
+    console.log("port_cache", this.port_cache)
+    console.log("port_mapping", this.port_mapping)
+
+  }
+
+  async refresh() {
+    this._mapping = {}
+    this.port_mapping = {}   // 127.0.0.1 => 192.168,..
+    this.local_network_mapping = {}
+
+    // 1. Localhost handling
+    this.localhost_home_router.handle()
+    this.localhost_variable_router.handle(this.kernel.memory.local)
+    let procs = await this.kernel.processes.refresh()
+    for(let proc of procs) {
+      this.localhost_port_router.handle(proc)
+    }
+
+    // 2. Peers handling
+    let peers = await this.kernel.peer.refresh()
+    for(let host in peers) {
+      let peer = peers[host]
+      this.peer_home_router.handle(peer)
+      this.peer_variable_router.handle(peer)
+      this.peer_port_router.handle(peer)
+    }
+    // replace :PORT_PLACEHOLDER with available ports
+
+    await this.fill()
+
+    console.log("CONFIG", JSON.stringify(this.config, null, 2))
+
+
+    let response = await axios.post('http://127.0.0.1:2019/load', this.config, {
+      headers: { 'Content-Type': 'application/json' }
+    }).catch((e) => {
+      console.log("EEEEEEEEEEEEE", e)
+    })
+    console.log("Caddy Response", { response })
+  }
+}
+module.exports = Router

package/kernel/router/localhost_home_router.js

@@ -0,0 +1,61 @@
+class LocalhostHomeRouter {
+  constructor(router) {
+    this.router = router
+  }
+  handle () {
+    this.router.config = {
+      "apps": {
+        "http": {
+          "servers": {
+            "main": {
+              "listen": [":443"],
+              "routes": [
+                {
+                  "match": [{ "method": ["OPTIONS"] }],
+                  "handle": [
+                    {
+                      "handler": "headers",
+                      "response": {
+                        "set": {
+                          "Access-Control-Allow-Origin": ["*"],
+                          "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                          "Access-Control-Allow-Headers": ["*"],
+                          "Vary": ["Origin"]
+                        }
+                      }
+                    },
+                    {
+                      "handler": "static_response",
+                      "status_code": 204
+                    }
+                  ]
+                },
+                {
+                  "match": [{ "host": [ this.router.default_match ] }],
+                  "handle": [
+                    {
+                      "handler": "reverse_proxy",
+                      "upstreams": [{ "dial": this.router.default_host + ":" + this.router.default_port }]
+                    },
+                    {
+                      "handler": "headers",
+                      "response": {
+                        "set": {
+                          "Access-Control-Allow-Origin": ["*"],
+                          "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+                          "Access-Control-Allow-Headers": ["*"],
+                          "Vary": ["Origin"]
+                        }
+                      }
+                    }
+                  ]
+                },
+              ]
+            }
+          }
+        }
+      }
+    }
+  }
+}
+module.exports = LocalhostHomeRouter

package/kernel/router/localhost_port_router.js

@@ -0,0 +1,15 @@
+const Common = require('./common')
+class LocalhostHomeRouter {
+  constructor(router) {
+    this.router = router
+    this.common = new Common(router)
+  }
+  handle (proc) {
+    this.common.handle({
+      match: `${proc.port}.localhost`,
+      dial: proc.ip,
+      host: this.router.kernel.peer.host,
+    })
+  }
+}
+module.exports = LocalhostHomeRouter

package/kernel/router/localhost_variable_router.js

@@ -0,0 +1,39 @@
+const path = require('path')
+const Common = require('./common')
+const Processor = require('./processor')
+class LocalhostVariableRouter extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+    this.common = new Common(router)
+  }
+  handle (local) {
+    console.log("localhost_variable_router", { local })
+    for(let script_path in local) {
+      let local_variables_for_script = local[script_path]
+      for(let key in local_variables_for_script) {
+        let val = local_variables_for_script[key]
+        if (val.startsWith("http")) {
+          let dial = val.replace(/https?:\/\//, '')
+          let name = this.api_name(script_path)
+          console.log({ dial, name, has_port: this.has_port(dial), parsed_dial: this.parse_ip(dial) })
+          if (this.has_port(dial)) {
+            if (key === "url") {
+              this.common.handle({
+                match: `${name}.localhost`.toLowerCase(),
+                dial,
+                host: this.router.kernel.peer.host,
+              })
+            }
+            this.common.handle({
+              match: `${key}.${name}.localhost`.toLowerCase(),
+              dial,
+              host: this.router.kernel.peer.host,
+            })
+          }
+        }
+      }
+    }
+  }
+}
+module.exports = LocalhostVariableRouter

package/kernel/router/peer_home_router.js

@@ -0,0 +1,15 @@
+const Common = require('./common')
+class PeerHomeRouter {
+  constructor(router) {
+    this.router = router
+    this.common = new Common(router)
+  }
+  handle(peer) {
+    this.common.handle({
+      match: `${this.router.default_prefix}.${peer.name}.${this.router.default_suffix}`,
+      dial: `${peer.host}:${this.router.default_port}`,
+      host: peer.host
+    })
+  }
+}
+module.exports = PeerHomeRouter

package/kernel/router/peer_port_router.js

@@ -0,0 +1,43 @@
+const path = require('path')
+const Processor = require('./processor')
+const Connector = require('./connector')
+class PeerPortRouter extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+    this.connector = new Connector(router)
+  }
+  handle(peer) {
+    let https_domains_mapping = this.router.info()[peer.host]
+    for(let dial in https_domains_mapping) {
+      let parsed_dial = this.parse_ip(dial)
+
+
+      let https_domains = https_domains_mapping[dial]
+
+      // 1. if https_domains is made up of one item <port>.localhost,
+      // add <port>.<account>.localhost
+      // 2. if https_domains is made up of two items <port>.localhost and <port>.<account>.localhost, don't do anything
+      for(let https_domain of https_domains) {
+        // insert the $account value right before the last part (1234.localhost => 1234.x.localhost)
+        let match
+        if (https_domain.endsWith(`${peer.name}.localhost`)) {
+          match = https_domain
+        } else {
+          let chunks = https_domain.split(".")
+          chunks.splice(chunks.length - 1, 0, peer.name)
+          match = chunks.join(".")
+        }
+        this.connector.handle({
+          match,
+          connector: {
+            host: peer.host,
+            port: parsed_dial.port
+          },
+          dial,
+        })
+      }
+    }
+  }
+}
+module.exports = PeerPortRouter

package/kernel/router/peer_variable_router.js

@@ -0,0 +1,48 @@
+const path = require('path')
+const Connector = require('./connector')
+const Processor = require('./processor')
+class PeerVariableRouter extends Processor {
+  constructor(router) {
+    super()
+    this.router = router
+    this.connector = new Connector(router)
+  }
+  handle(peer) {
+    for(let script_path in peer.local) {
+      let local_variables = peer.local[script_path]
+      for(let key in local_variables) {
+        let val = local_variables[key]
+        console.log({ key, val })
+        if (val.startsWith("http")) {
+          let dial = val.replace(/https?:\/\//, '')
+          let name = this.api_name(script_path)
+          console.log({ dial, name, has_port: this.has_port(dial), parsed_dial: this.parse_ip(dial) })
+          if (this.has_port(dial)) {
+            let parsed_dial = this.parse_ip(dial)
+            if (key === "url") {
+              this.connector.handle({
+                match: `${name}.${peer.name}.localhost`.toLowerCase(),
+                connector: {
+                  host: peer.host,
+                  port: parsed_dial.port
+                },
+                dial,
+                host: peer.host,
+              })
+            }
+            connector({
+              match:`${key}.${name}.${peer.name}.localhost`.toLowerCase(),
+              connector: {
+                host: peer.host,
+                port: parsed_dial.port
+              },
+              dial,
+              host: peer.host,
+            })
+          }
+        }
+      }
+    }
+  }
+}
+module.exports = PeerVariableRouter

package/kernel/router/processor.js

@@ -0,0 +1,18 @@
+const path = require('path')
+class Processor {
+  has_port(str) {
+    let chunks = str.split(":")
+    return chunks.length === 2
+  }
+  api_name(str) {
+    let api_path = this.router.kernel.path("api")
+    let rel_path = path.relative(api_path, str)
+    console.log({ api_path, rel_path, })
+    return rel_path.split(path.sep)[0]
+  }
+  parse_ip(str) {
+    let chunks = str.split(":")
+    return { host: chunks[0], port: chunks[1] }
+  }
+}
+module.exports = Processor

package/kernel/router/root.js

@@ -0,0 +1,32 @@
+module.exports = (payload, { match, dial }) => {
+  // create a root level reverse proxy for
+  //  - peer with ip,
+  //  - at port
+  // The ip is only used for ensuring uniqueness
+  // But the only thing that will be exposed is the port
+  let name = match.ip + ":" + match.port
+  payload.apps.http.servers[name] = {
+    // 0.0.0.0:<new_port>
+    "listen": ["0.0.0.0:PORT_PLACEHOLDER_" + name],
+    "routes": [{
+      "handle": [
+        {
+          "handler": "reverse_proxy",
+          // create a route from <new_port> to dial
+          "upstreams": [{ "dial": dial }]
+        },
+        {
+          "handler": "headers",
+          "response": {
+            "set": {
+              "Access-Control-Allow-Origin": ["*"],
+              "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+              "Access-Control-Allow-Headers": ["*"],
+              "Vary": ["Origin"]
+            }
+          }
+        }
+      ]
+    }]
+  }
+}

package/kernel/sysinfo.js

@@ -75,7 +75,6 @@ class Sysinfo {
     this.info.gpus = gpus
     this.info.gpu = gpu
     this.info.gpu_model = gpu_model
-    console.log("> this.info", this.info)
 
   }
 //  async time() {

package/kernel/util.js

@@ -271,7 +271,6 @@ const find_venv = async (root) => {
 }
 
 const update_env = async (filepath, changes) => {
-  console.log("UPDATE_ENV", { filepath, changes })
   const env = await fs.promises.readFile(filepath, "utf8")
   let append = false;
   const newval = Object.keys(changes).reduce((result, varname) => {
@@ -308,7 +307,7 @@ const update_env = async (filepath, changes) => {
   await fs.promises.writeFile(filepath, newval)
 };
 function fill_object(obj, pattern, list, cache) {
-  console.log("FILL_OBJECT")
+  console.log("FILL_OBJECT", { obj, pattern, list, cache })
   const map = {};
   const replaced_map = cache || {}
   let index = 0;
@@ -317,7 +316,6 @@ function fill_object(obj, pattern, list, cache) {
     if (typeof value === 'string') {
       return value.replace(pattern, (match) => {
         if (!(match in map)) {
-          console.log( "fill_object", { replaced_map, match })
           if (replaced_map[match]) {
             map[match] = replaced_map[match]
           } else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "3.8.44",
+  "version": "3.8.45",
   "description": "",
   "main": "index.js",
   "scripts": {

package/pipe/index.js

@@ -22,7 +22,6 @@ class Pipe {
   }
   // a pipe server for creating publicly shareable endpoints
   async start(url, scriptPath, passcode, config) {
-    console.log("pipe.start", { url, scriptPath, passcode, config })
 
     let port = await this.kernel.port()
     let app = express();
@@ -100,7 +99,6 @@ class Pipe {
 
 
     await new Promise((resolve, reject) => {
-      console.log("starting pipe server at port", port)
       let server = app.listen(port, () => {
         console.log(`Pipe server listening on port ${port}`)
         resolve()