pinokiod 3.8.147 → 3.8.149

package/kernel/loader.js

@@ -40,30 +40,32 @@ class Loader {
           resolved = null
         }
       } catch (e) {
-        console.log(`[load error] ${_path}`)
         resolved = null
       }
     }
+    if (!resolved) {
+      console.log(`[did not load] ${_path}`)
+    }
     return { resolved, extension, dirname }
   }
   async requireJSON(filepath) {
     let config
     try { config = require(filepath) } catch (e) {
-      console.log(`[load error] ${filepath}`)
+//      console.log(`[did not load] ${filepath}`)
     }
     return config
   }
   async requireJS(filepath) {
     let config
     try { config = require(filepath) } catch (e) {
-      console.log(`[load error] ${filepath}`)
+//      console.log(`[did not load] ${filepath}`)
     }
     try {
       // if the required module is a class, return the instantiated object
       return new config();
     } catch (e) {
       // otherwise return normally
-      console.log(`[load error] ${filepath}`)
+//      console.log(`[did not load] ${filepath}`)
       return config
     }
   }

package/kernel/peer.js

@@ -86,7 +86,7 @@ class PeerDiscovery {
     }
   }
   async _broadcast(host) {
-    console.log("peer _broadcast", host)
+//    console.log("peer _broadcast", host)
     try {
       let res = await axios.post(`http://${host}:${this.default_port}/pinokio/peer/refresh`, {
         timeout: 2000
@@ -108,6 +108,7 @@ class PeerDiscovery {
   // refresh peer info
   async refresh(peers) {
     if (this.active) {
+      this.refreshing = true
       let refresh_peers
       if (peers) {
         refresh_peers = peers
@@ -118,7 +119,7 @@ class PeerDiscovery {
       let peer_info = await Promise.all(refresh_peers.map((host) => {
         return this._refresh(host)
       }))
-      console.log({ peer_info })
+//      console.log({ peer_info })
       for(let i=0; i<peer_info.length; i++) {
         let peer = peer_info[i]
         if (peer) {
@@ -128,7 +129,8 @@ class PeerDiscovery {
           }
         }
       }
-      console.log("REFRESH", this.info)
+//      console.log("REFRESH", this.info)
+      this.refreshing = false
       return this.info
     }
   }

package/kernel/procs.js

@@ -98,6 +98,7 @@ class Procs {
             const fullname = pidToName[pid] || 'Unknown';
             const name = fullname.split(path.sep).pop()
             if (["caddy", "caddy.exe"].includes(name)) {
+              this.caddy_pid = pid
               return null
             } else {
               map["" + pid] =  fullname
@@ -107,7 +108,7 @@ class Procs {
 //          list = list.filter((item) => {
 //            return !["caddy", "caddy.exe"].includes(item.name.toLowerCase()) 
 //          })
-          console.log("#LIST", { list, size: list.length })
+//          console.log("#LIST", { list, size: list.length })
           resolve(list)
         });
       });      

package/kernel/prototype.js

@@ -1,5 +1,6 @@
 const fs = require('fs')
 const path = require('path')
+const { glob, sync, hasMagic } = require('glob-gitignore')
 class Proto {
   constructor(kernel) {
     this.kernel = kernel
@@ -10,24 +11,26 @@ class Proto {
     let exists = await this.kernel.exists("prototype")
     if (!exists) {
       console.log("prototype doesn't exist. cloning...")
+      await fs.promises.mkdir(this.kernel.path("prototype"), { recursive: true }).catch((e) => { })
       await this.kernel.exec({
-        message: "git clone https://github.com/peanutcocktail/prototype",
-        path: this.kernel.homedir
+        message: "git clone https://github.com/peanutcocktail/prototype system",
+        path: this.kernel.path("prototype")
       }, (e) => {
         process.stdout.write(e.raw)
       })
     }
-    // get all prototypes
+
+
+    let cwd = path.resolve(this.kernel.homedir, "prototype")
+    let prototype_paths = (await glob('**/pinokio.js', { cwd }))
     let prototype_dir = path.resolve(this.kernel.homedir, "prototype")
-    let prototypes = await fs.promises.readdir(prototype_dir, { withFileTypes: true })
-    prototypes = prototypes.filter(d => d.isDirectory()).map(d => d.name);
-    console.log({ prototypes })
     this.items = []
     this.kv = {}
-    for(let proto of prototypes) {
-      let pinokiojs = path.resolve(prototype_dir, proto, "pinokio.js")
+    for(let prototype_path of prototype_paths) {
+      let proto = path.dirname(prototype_path)
+      let pinokiojs = path.resolve(prototype_dir, prototype_path)
       let config = await this.kernel.require(pinokiojs)
-      if (config && config.type === "prototype") {
+      if (config && config.create && typeof config.create === "function") {
         if (config.icon) {
           config.icon = "/prototype/" + proto + "/" + config.icon
         } else {

package/kernel/router/common.js

@@ -1,50 +1,59 @@
-class Common {
+const Processor = require('./processor')
+class Common extends Processor {
   constructor (router) {
+    super()
     this.router = router
   }
   handle({ match, dial, host }) {
-    this.router.config.apps.http.servers.main.routes.push({
-      "match": [{ "host": [match] }],
-      "handle": [
-        {
-          "handler": "reverse_proxy",
-          "upstreams": [{ "dial": dial }],
-          "headers": {
-            "request": {
-              "set": {
-                "Origin": ["http://127.0.0.1:11434"],
-                "X-Forwarded-Proto": ["https"],
-                "X-Forwarded-Host": ["{http.request.host}"]
-              }
-            },
-            "response": {
-              "set": {
-                "Access-Control-Allow-Origin": ["*"],
-                "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-                "Access-Control-Allow-Headers": ["*"],
-                "Vary": ["Origin"]
-              }
-            }
+    let handler = [{
+      "handler": "reverse_proxy",
+      "upstreams": [{ "dial": dial }],
+      "headers": {
+        "request": {
+          "set": {
+//            "Origin": ["http://127.0.0.1:11434"],
+            "X-Forwarded-Proto": ["https"],
+            "X-Forwarded-Host": ["{http.request.host}"]
           }
         },
-        //{
-        //  "handler": "headers",
-        //  "request": {
-        //    "set": {
-        //      "X-Forwarded-Proto": ["https"],
-        //      "X-Forwarded-Host": ["{http.request.host}"]
-        //    }
-        //  },
-        //  "response": {
-        //    "set": {
-        //      "Access-Control-Allow-Origin": ["*"],
-        //      "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-        //      "Access-Control-Allow-Headers": ["*"],
-        //      "Vary": ["Origin"]
-        //    }
-        //  }
-        //}
-      ]
+        "response": {
+          "set": {
+            "Access-Control-Allow-Origin": ["*"],
+            "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+            "Access-Control-Allow-Headers": ["*"],
+            "Vary": ["Origin"]
+          }
+        }
+      }
+    },
+    //{
+    //  "handler": "headers",
+    //  "request": {
+    //    "set": {
+    //      "X-Forwarded-Proto": ["https"],
+    //      "X-Forwarded-Host": ["{http.request.host}"]
+    //    }
+    //  },
+    //  "response": {
+    //    "set": {
+    //      "Access-Control-Allow-Origin": ["*"],
+    //      "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+    //      "Access-Control-Allow-Headers": ["*"],
+    //      "Vary": ["Origin"]
+    //    }
+    //  }
+    //}
+    ]
+
+    // if the dial port has been overridden by router.custom_routers, use that instead
+    let parsed_dial = this.parse_ip(dial)
+    let override_handler = this.router.custom_routers[String(parsed_dial.port)]
+    if (override_handler) {
+      handler = override_handler
+    }
+    this.router.config.apps.http.servers.main.routes.push({
+      "match": [{ "host": [match] }],
+      "handle": handler
     })
     this.router.add({ host, dial, match })
   }

package/kernel/router/connector.js

@@ -1,41 +1,39 @@
-class Connector {
+const Processor = require('./processor')
+class Connector extends Processor {
   constructor(router) {
+    super()
     this.router = router
   }
   handle ({ match, connector, dial, }) {
-    console.log("connector.handle", { match, connector, dial })
+//    console.log("connector.handle", { match, connector, dial })
     let connector_name = connector.host + ":" + connector.port
     let listener = "0.0.0.0:PORT_PLACEHOLDER_" + connector_name
     let listener_pointer = connector.host + ":PORT_PLACEHOLDER_" + connector_name
-    console.log("CONNECTING", { listener, dial, listener_pointer, match })
-    if(!this.router.config.apps.http.servers[connector_name]) {
-      this.router.config.apps.http.servers[connector_name] = {
-        "listen": [listener],
-        "routes": [{
-          "handle": [
-            {
-              "handler": "reverse_proxy",
-              "upstreams": [{ "dial": dial }],
-              "headers": {
-                "request": {
-                  "set": {
-                    "Origin": ["http://127.0.0.1:11434"],
-                    "X-Forwarded-Proto": ["{http.request.header.X-Forwarded-Proto}"],
-                    //"X-Forwarded-Proto": ["{http.request.scheme}"],
-                    //"X-Forwarded-Proto": ["https"],
-                    //"X-Forwarded-Host": ["{http.request.host}"]
-                  }
-                },
-                "response": {
-                  "set": {
-                    "Access-Control-Allow-Origin": ["*"],
-                    "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-                    "Access-Control-Allow-Headers": ["*"],
-                    "Vary": ["Origin"]
-                  }
-                }
-              }
-            },
+    let port_placeholder = "PORT_PLACEHOLDER_" + connector_name
+//    console.log("CONNECTING", { listener, dial, listener_pointer, match })
+
+    let handler = [{
+      "handler": "reverse_proxy",
+      "upstreams": [{ "dial": dial }],
+      "headers": {
+        "request": {
+          "set": {
+//            "Origin": ["http://127.0.0.1:11434"],
+            "X-Forwarded-Proto": ["{http.request.header.X-Forwarded-Proto}"],
+            //"X-Forwarded-Proto": ["{http.request.scheme}"],
+            //"X-Forwarded-Proto": ["https"],
+            //"X-Forwarded-Host": ["{http.request.host}"]
+          }
+        },
+        "response": {
+          "set": {
+            "Access-Control-Allow-Origin": ["*"],
+            "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+            "Access-Control-Allow-Headers": ["*"],
+            "Vary": ["Origin"]
+          }
+        }
+      }
 //            {
 //              "handler": "headers",
 //              "request": {
@@ -53,7 +51,18 @@ class Connector {
 //                }
 //              }
 //            }
-          ]
+    }]
+    // if the dial port has been overridden by router.custom_routers, use that instead
+    let parsed_dial = this.parse_ip(dial)
+    let override_handler = this.router.custom_routers[String(parsed_dial.port)]
+    if (override_handler) {
+      handler = override_handler
+    }
+    if(!this.router.config.apps.http.servers[connector_name]) {
+      this.router.config.apps.http.servers[connector_name] = {
+        "listen": [listener],
+        "routes": [{
+          "handle": handler
         }]
       }
     }
@@ -103,8 +112,9 @@ class Connector {
 //        }
       ]
     })
-    console.log("Router.add", { host: connector.host, dial: listener_pointer, match })
     this.router.add({ host: connector.host, dial: listener_pointer, match })
+//    this.router._tree[connector_name] = dial
+//    this.router._tree[connector_name] = dial
   }
 }
 module.exports = Connector

package/kernel/router/index.js

@@ -1,5 +1,6 @@
 const axios = require('axios')
 const path = require('path')
+const { glob, sync, hasMagic } = require('glob-gitignore')
 const Util = require('../util')
 const LocalhostHomeRouter = require('./localhost_home_router')
 const LocalhostVariableRouter = require('./localhost_variable_router')
@@ -28,6 +29,47 @@ class Router {
     this.mapping = {}
     this.port_mapping = {}   // 127.0.0.1 => 192.168,..
     this.local_network_mapping = {}
+    this.custom_routers = {}
+  }
+  async init() {
+    console.log("router.init")
+    // if ~/pinokio/network doesn't exist, clone
+    let exists = await this.kernel.exists("network/system")
+    if (!exists) {
+      console.log("network doesn't exist. cloning...")
+      await fs.promises.mkdir(this.kernel.path("network"), { recursive: true }).catch((e) => { })
+      await this.kernel.exec({
+        message: "git clone https://github.com/peanutcocktail/network system",
+        path: this.kernel.path("network")
+      }, (e) => {
+        process.stdout.write(e.raw)
+      })
+    }
+
+
+    let cwd = path.resolve(this.kernel.homedir, "network")
+    let router_paths = (await glob('**/*.json', { cwd }))
+    let router_dir = path.resolve(this.kernel.homedir, "network")
+
+
+    // create a custom_map that maps port to custom router declarations
+    /*
+      custom_routers = {
+        PORT1: <caddy handler>,
+        PORT2: <caddy handler>,
+        ...
+      }
+    */
+    this.custom_routers = {}
+    console.log({ router_paths })
+    for(let router_path of router_paths) {
+      let router_abs_path = this.kernel.path("network", router_path)
+      let config = await this.kernel.require(router_abs_path)
+      console.log({ config })
+      for(let port in config) {
+        this.custom_routers[port] = config[port]
+      }
+    }
   }
   async add({ host, match, dial }) {
     if (!this._mapping[host]) {
@@ -50,7 +92,6 @@ class Router {
       }
     }
 
-    console.log("BEFORE REPLACING", mapping)
 
     let host_mapping = mapping[this.kernel.peer.host]
     if (host_mapping) {
@@ -60,7 +101,7 @@ class Router {
             let transformed = key.replace(":" + cache_key, '')
             let port = this.port_cache[cache_key]
             host_mapping[transformed + ":" + port] = host_mapping[key]
-            console.log("replacing", transformed + ":" + port, "with", host_mapping[key])
+//            console.log("replacing", transformed + ":" + port, "with", host_mapping[key])
             delete host_mapping[key]
           }
         }
@@ -75,9 +116,10 @@ class Router {
 //      let m = JSON.parse(json)
 //
 //      mapping[this.kernel.peer.host] = m
-      console.log("AFTER REPLACING", mapping)
+//      console.log("AFTER REPLACING", mapping)
     }
 
+
     return mapping
 
 //    let json = JSON.stringify(mapping)
@@ -105,17 +147,17 @@ class Router {
   */
   published() {
     let pub = {}
-    console.log("published info", this.info)
+//    console.log("published info", this.info)
     if (this.info) {
       let routes = this.info[this.kernel.peer.host]
       for(let dial in routes) {
         let matches = routes[dial]
         // current host => the single source of truth
-        if (dial.startsWith(this.kernel.peer.host)) {
+//        if (dial.startsWith(this.kernel.peer.host)) {
           pub[dial] = matches
-        }
+//        }
       }
-      console.log({ pub })
+//      console.log({ pub })
     }
     return pub
   }
@@ -143,14 +185,10 @@ class Router {
     }
     */
 
-
-    console.log("^^^^^^^^", { count , matches })
-
     let new_config
  
     if (count > 0) {
       let ports = await this.kernel.ports(count)
-      console.log("Filling with port cache", JSON.stringify({ port_cache: this.port_cache, config: this.config }, null, 2))
       let response = Util.fill_object(this.config, /PORT_PLACEHOLDER_.+/, ports, this.port_cache)
       this.config = response.result
       let replaced_map = response.replaced_map
@@ -170,7 +208,7 @@ class Router {
 
         //let original_port = key.replace("PORT_PLACEHOLDER_", "")
         this.port_mapping[original_port] = replaced_map[key]
-        console.log("set port_mapping", { original_port, key, replaced_map })
+//        console.log("set port_mapping", { original_port, key, replaced_map })
       }
       this.port_cache = replaced_map
     }
@@ -181,122 +219,114 @@ class Router {
   // set local config
   async local() {
     this._mapping = {}
-    console.log("Processing local")
+//    this._tree = {}
     this.localhost_home_router.handle()
-    console.log("after 1", this._mapping)
     this.localhost_variable_router.handle(this.kernel.memory.local)
-    console.log("after 2", this._mapping)
     for(let proc of this.kernel.processes.info) {
       this.localhost_port_router.handle(proc)
     }
-    console.log("after 3", this._mapping)
     //this.mapping = this._mapping
 
 
     for(let host in this.kernel.peer.info) {
       let peer = this.kernel.peer.info[host]
       if (peer.host === this.kernel.peer.host) {
-        console.log("peer.host === kernel.peer.host", peer.host)
         this.peer_home_router.handle(peer)
-        console.log("after 4", this._mapping)
         this.peer_variable_router.handle(peer)
-        console.log("after 5", this._mapping)
         this.peer_port_router.handle(peer)
-        console.log("after 6", this._mapping)
       }
     }
-    console.log("after 7", this._mapping)
     await this.fill()
-    console.log("after 8", this._mapping)
     this.mapping = this._mapping
+//    this.tree = this._tree
   }
 
   // set remote config
   async remote() {
-    console.log("Before remote", this.kernel.peer.info)
     for(let host in this.kernel.peer.info) {
       let peer = this.kernel.peer.info[host]
       if (peer.host !== this.kernel.peer.host) {
         this.peer_peer_router.handle(peer)
       }
     }
-    console.log("After remote", this.kernel.peer.info)
     //await this.fill()
     this.mapping = this._mapping
+//    this.tree = this._tree
     this.info = this._info()
+//    console.log("TREE", JSON.stringify(this.tree, null, 2))
   }
 
   // update caddy config
   async update() {
-//    console.log("this.config", JSON.stringify(this.config))
-//    console.log("this.old_config", JSON.stringify(this.old_config))
     if (JSON.stringify(this.config) === JSON.stringify(this.old_config)) {
-      console.log("config hasn't updated")
+//      console.log("config hasn't updated")
     } else {
-      console.log("config has updated")
-      let response = await axios.post('http://127.0.0.1:2019/load', this.config, {
-        headers: { 'Content-Type': 'application/json' }
-      }).catch((e) => {
+      console.log("caddy config has updated", this.config)
+      try {
+        let response = await axios.post('http://127.0.0.1:2019/load', this.config, {
+          headers: { 'Content-Type': 'application/json' }
+        })
+        console.log("Caddy Response", { response })
+        this.old_config = this.config
+      } catch (e) {
         console.log("EEEEEEEEEEEEE", e.message)
-      })
-//      console.log("Caddy Response", { response })
-      this.old_config = this.config
-    }
-  }
-
-  async refresh() {
-//    console.log("router.refresh")
-//    console.time("router.refresh")
-//    console.log("> procs", this.kernel.processes.info)
-//    console.log("> peers", this.kernel.peer.info)
-    this._mapping = {}
-//    this.port_mapping = {}   // 127.0.0.1 => 192.168,..
-//    this.local_network_mapping = {}
-
-    let old_config = this.config
-
-    // 1. Localhost handling
-    this.localhost_home_router.handle()
-    this.localhost_variable_router.handle(this.kernel.memory.local)
-    for(let proc of this.kernel.processes.info) {
-      this.localhost_port_router.handle(proc)
-    }
-    this.mapping = this._mapping
-
-    // 2. Peers handling
-    for(let host in this.kernel.peer.info) {
-      let peer = this.kernel.peer.info[host]
-      if (peer.host === this.kernel.peer.host) {
-        this.peer_home_router.handle(peer)
-        this.peer_variable_router.handle(peer)
-        this.peer_port_router.handle(peer)
-      } else {
-        this.peer_peer_router.handle(peer)
       }
     }
-    // replace :PORT_PLACEHOLDER with available ports
-
-
-    await this.fill()
-    this.mapping = this._mapping
-    console.timeEnd("router.refresh")
-
-
-    // compare old config with new config
-    // if it hasn't changed, don't do anything
-    // if it has changed, refresh
-
-    if (JSON.stringify(this.config) === JSON.stringify(old_config)) {
-      console.log("config hasn't updated")
-    } else {
-      console.log("config has updated")
-      let response = await axios.post('http://127.0.0.1:2019/load', this.config, {
-        headers: { 'Content-Type': 'application/json' }
-      }).catch((e) => {
-        console.log("EEEEEEEEEEEEE", e.message)
-      })
-      console.log("Caddy Response", { response })
-    }
   }
+
+//  async refresh() {
+////    console.log("router.refresh")
+////    console.time("router.refresh")
+////    console.log("> procs", this.kernel.processes.info)
+////    console.log("> peers", this.kernel.peer.info)
+//    this._mapping = {}
+////    this.port_mapping = {}   // 127.0.0.1 => 192.168,..
+////    this.local_network_mapping = {}
+//
+//    let old_config = this.config
+//
+//    // 1. Localhost handling
+//    this.localhost_home_router.handle()
+//    this.localhost_variable_router.handle(this.kernel.memory.local)
+//    for(let proc of this.kernel.processes.info) {
+//      this.localhost_port_router.handle(proc)
+//    }
+//    this.mapping = this._mapping
+//
+//    // 2. Peers handling
+//    for(let host in this.kernel.peer.info) {
+//      let peer = this.kernel.peer.info[host]
+//      if (peer.host === this.kernel.peer.host) {
+//        this.peer_home_router.handle(peer)
+//        this.peer_variable_router.handle(peer)
+//        this.peer_port_router.handle(peer)
+//      } else {
+//        this.peer_peer_router.handle(peer)
+//      }
+//    }
+//    // replace :PORT_PLACEHOLDER with available ports
+//
+//
+//    await this.fill()
+//    this.mapping = this._mapping
+//    console.timeEnd("router.refresh")
+//
+//
+//    // compare old config with new config
+//    // if it hasn't changed, don't do anything
+//    // if it has changed, refresh
+//
+//    if (JSON.stringify(this.config) === JSON.stringify(old_config)) {
+//      console.log("config hasn't updated")
+//    } else {
+//      console.log("config has updated")
+//      let response = await axios.post('http://127.0.0.1:2019/load', this.config, {
+//        headers: { 'Content-Type': 'application/json' }
+//      }).catch((e) => {
+//        console.log("EEEEEEEEEEEEE", e.message)
+//      })
+//      console.log("Caddy Response", { response })
+//    }
+//  }
 }
 module.exports = Router

package/kernel/router/localhost_home_router.js

@@ -3,6 +3,7 @@ class LocalhostHomeRouter {
     this.router = router
   }
   handle () {
+    this.router.add({ host: this.router.kernel.peer.host, dial: this.router.default_host + ":" + this.router.default_port, match: this.router.default_match })
     this.router.config = {
       "apps": {
         "http": {

package/kernel/router/localhost_port_router.js

@@ -5,7 +5,6 @@ class LocalhostPortRouter {
     this.common = new Common(router)
   }
   handle (proc) {
-    console.log("LocalhostPortRouter", proc)
     this.common.handle({
       match: `${proc.port}.localhost`,
       dial: proc.ip,