pinokiod 3.8.148 → 3.8.150

package/kernel/environment.js

@@ -439,4 +439,53 @@ const get_raw = async (homedir) => {
   }
   return current_env
 }
-module.exports = { ENV, get, get2, init_folders }
+
+const requirements = async (script, filepath, kernel) => {
+  let pre_items = []
+  let requires_instantiation = false
+  if (script && script.pre) {
+    let env = await get2(filepath, kernel)
+    for(let item of script.pre) {
+      if (item.env) {
+        // if index is not set, use 0 as default (for key)
+        if (!item.index) {
+          item.index = 0
+        }
+        if (item.key) {
+          const hasProtocol = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(item.key);
+          const url = new URL(hasProtocol ? item.key : `https://${item.key}`);
+          item.host = url.host
+          item.val = await kernel.kv.get(item.host, item.index)
+        } else {
+          item.host = ""
+        }
+
+        if (env[item.env]) {
+          item.val = env[item.env]
+        } else {
+          if (item.default) {
+            if (typeof item.default === 'object') {
+              if (item.default.key) {
+                const hasProtocol = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(item.default.key);
+                const url = new URL(hasProtocol ? item.default.key : `https://${item.default.key}`);
+                item.host = url.host
+
+                let index = item.default.index || 0
+                item.index = index
+                item.val = await kernel.kv.get(item.host, index)
+              } else {
+                item.host = ""
+              }
+            } else {
+              item.val = item.default
+            }
+          }
+          requires_instantiation = true
+        }
+        pre_items.push(item)
+      }
+    }
+  }
+  return { items: pre_items, requires_instantiation }
+}
+module.exports = { ENV, get, get2, init_folders, requirements }

package/kernel/index.js

@@ -118,21 +118,6 @@ class Kernel {
       }
     }
   }
-//  path_exists(...args) {
-//    return this.info.exists(...args)
-//  }
-//  path_resolve(...args) {
-//    return this.info.path(...args)
-//  }
-//  script_running(...args) {
-//    return this.info.running(...args)
-//  }
-//  script_local(...args) {
-//    return this.info.local(...args)
-//  }
-//  script_global(...args) {
-//    return this.info.global(...args)
-//  }
   async env(...args) {
     let folderpath
     if (args) {
@@ -267,55 +252,40 @@ class Kernel {
       active = true
     }
 //    console.log("kernel.refresh", { active, notify_peers })
-    if (active) {
+
+    let installed = await this.bin.check_installed({ name: "caddy" })
+    console.log("caddy installed?", installed)
+    if (installed && active) {
 
       // 1. get the process list
-//      console.log("Step 1: Get process list")
       await this.processes.refresh()
-//      console.log("Process info", this.processes.info)
 
       // 2. refresh peer info to reflect the proc info
       await this.peer.refresh()
 
-      // 3. set current local host router info
-//      console.log("Step 3: Set current local host router")
+      // 3. load custom routers from ~/pinokio/network
+      await this.router.init()
+
+      // 4. set current local host router info
       await this.router.local()
-//      console.log("router mapping after step 3", this.router.info)
 
-      // 4. refresh peer info to reflect the updated router info
+      // 5. refresh peer info to reflect the updated router info
       await this.peer.refresh()
 
-      // 4. tell peers to refresh
+      // 6. tell peers to refresh
       if (notify_peers) {
-//        console.log("Step 4: notify peers to refresh")
         await this.peer.notify_peers()
       }
 
-      // 4. update remote router
-//      console.log("Step 5: Set remote routers")
+      // 7. update remote router
       await this.router.remote()
-//      console.log("router mapping after step 5", this.router.info)
 
+      // 8. update caddy config
       await this.router.update()
 
+      // 9. announce self to the peer network
+      this.peer.announce()
 
-//      await Promise.all([
-//        // refresh proc info
-//        this.processes.refresh(),
-//        // refresh peer info
-//        this.peer.refresh()
-//      ])
-//      console.log("proc info", this.processes.info)
-//      console.log("peer info", this.peer.info)
-//      // refresh router info
-//      await this.router.refresh()
-//      console.log("router info", this.router.published())
-////      await this.peer.refresh()
-//
-//      if (notify_peers) {
-//        // broadcast to peers to refresh
-//        await this.peer.notify_peers()
-//      }
     }
   }
   async clearLog(group) {
@@ -528,25 +498,13 @@ class Kernel {
     let home = this.store.get("home")
 
     // reset shells if they exist
-    console.log("KERNEL INIT")
     if (this.shell) {
-      console.log("> shell reset")
       this.shell.reset()
     }
     if (this.peer) {
-      console.log("> peer stop")
       this.peer.stop()
     }
 
-//    // kill all processes just in case
-//    await new Promise((resolve, reject) => {
-//      console.log("killing child processes for", process.pid)
-//      kill(process.pid, {}, 'SIGKILL', () => {
-//        console.log("child procs killed for", process.pid)
-//        resolve()
-//      });
-//    })
-
     this.info = new Info(this)
     this.pipe = new Pipe(this)
     this.proto = new Proto(this)
@@ -668,10 +626,8 @@ class Kernel {
       this.bin.init().then(() => {
         if (this.homedir) {
           this.shell.init().then(async () => {
-            this.bin.check_bin()
-            console.log(">>>> 1")
+//            this.bin.check()
             if (this.envs) {
-              console.log(">>>> 2")
               this.template.update({
                 env: this.envs,
                 envs: this.envs,
@@ -720,7 +676,6 @@ class Kernel {
                         }
                       }
                     }
-                console.log(">>>>>> d")
                     clearInterval(interval)
                     setTimeout(() => {
                       this.launch_complete = true

package/kernel/info.js

@@ -40,16 +40,13 @@ class Info {
   running(...args) {
     let cwd = path.dirname(this.caller())
     let resolved_path = path.resolve(cwd, ...args)
-    console.log("api.running", this.kernel.api.running)
     let running = this.kernel.api.running[resolved_path]
-    console.log("info.running", { cwd, resolved_path, running })
     return running ? running : false
   }
   exists(...args) {
     let cwd = path.dirname(this.caller())
     let resolved_path = path.resolve(cwd, ...args)
     let exists = fs.existsSync(resolved_path)
-    console.log("info.exists", { cwd, resolved_path, exists })
     return exists ? exists: false
   }
   local(...args) {

package/kernel/kv.js

@@ -12,6 +12,9 @@ class KV {
         json[host] = []
       }
       // if index doesn't exist, set index 0
+      if (!index) {
+        index = 0
+      }
       json[host][index] = val
       await fs.promises.writeFile(filePath, JSON.stringify(json, null, 2), 'utf8')
     }

package/kernel/loader.js

@@ -6,14 +6,6 @@ const clearModule = require('clear-module');
 
 class Loader {
   async load(_path) {
-
-//    let exists = await new Promise(r=>fs.access(_path, fs.constants.F_OK, e => r(!e)))
-//    if (!exists) {
-//      console.log(`[load] ${_path} does not exist`)
-//      return { resolved: null }
-//    }
-
-
     let resolved
     let dirname
     let extension = path.extname(_path)
@@ -40,30 +32,29 @@ class Loader {
           resolved = null
         }
       } catch (e) {
-        console.log(`[did not load] ${_path}`)
         resolved = null
       }
     }
+    if (!resolved) {
+      console.log(`[did not load] ${_path}`)
+    }
     return { resolved, extension, dirname }
   }
   async requireJSON(filepath) {
     let config
     try { config = require(filepath) } catch (e) {
-      console.log(`[did not load] ${filepath}`)
     }
     return config
   }
   async requireJS(filepath) {
     let config
     try { config = require(filepath) } catch (e) {
-      console.log(`[did not load] ${filepath}`)
     }
     try {
       // if the required module is a class, return the instantiated object
       return new config();
     } catch (e) {
       // otherwise return normally
-      console.log(`[did not load] ${filepath}`)
       return config
     }
   }

package/kernel/peer.js

@@ -10,6 +10,7 @@ class PeerDiscovery {
     this.peers = new Set();
     this.host = this._getLocalIPAddress()
     this.default_port = 42000
+    this.peers.add(this.host)
 //    this.start();
   }
   stop() {
@@ -19,6 +20,9 @@ class PeerDiscovery {
       this.socket.close()
     }
   }
+  announce() {
+    this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
+  }
   async start(kernel) {
     let env = await Environment.get(kernel.homedir)
 
@@ -36,12 +40,9 @@ class PeerDiscovery {
 
     if (this.active) {
       // Listen for incoming pings
-      console.log("kernel.peer start")
       this.socket = dgram.createSocket('udp4');
       this.socket.on('message', (msg, rinfo) => {
-//        console.log("peer message", { msg, rinfo })
         const ip = rinfo.address;
-//        console.log({rinfo})
         if (msg.toString() === this.message.toString() && this._isLocalLAN(ip)) {
           if (!this.peers.has(ip)) {
             console.log(`Discovered peer: ${ip}`);
@@ -62,16 +63,13 @@ class PeerDiscovery {
 
         this.socket.setBroadcast(true);
 
-        // Send broadcast pings every interval
-        this.interval_handle = setInterval(() => {
-//          console.log("send message to peers")
-          //this.socket.send(this.message, 0, this.message.length, this.port, '127.0.0.1');
-          //console.log("sent to 127.0.0.1")
-          //this.socket.send(this.message, 0, this.message.length, this.port, '255.255.255.255');
-          this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
-          //console.log("sent to 255.255.255.255")
-//          console.log("sent message")
-        }, this.interval);
+        this.announce()
+
+//        // Send broadcast pings every interval
+//        this.interval_handle = setInterval(() => {
+//          //this.socket.send(this.message, 0, this.message.length, this.port, '255.255.255.255');
+//          this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
+//        }, this.interval);
       });
     }
   }
@@ -86,7 +84,6 @@ class PeerDiscovery {
     }
   }
   async _broadcast(host) {
-//    console.log("peer _broadcast", host)
     try {
       let res = await axios.post(`http://${host}:${this.default_port}/pinokio/peer/refresh`, {
         timeout: 2000
@@ -119,7 +116,6 @@ class PeerDiscovery {
       let peer_info = await Promise.all(refresh_peers.map((host) => {
         return this._refresh(host)
       }))
-//      console.log({ peer_info })
       for(let i=0; i<peer_info.length; i++) {
         let peer = peer_info[i]
         if (peer) {
@@ -129,7 +125,6 @@ class PeerDiscovery {
           }
         }
       }
-//      console.log("REFRESH", this.info)
       this.refreshing = false
       return this.info
     }

package/kernel/procs.js

@@ -98,16 +98,13 @@ class Procs {
             const fullname = pidToName[pid] || 'Unknown';
             const name = fullname.split(path.sep).pop()
             if (["caddy", "caddy.exe"].includes(name)) {
+              this.caddy_pid = pid
               return null
             } else {
               map["" + pid] =  fullname
               return { port, pid , name, fullname, ip }
             }
           }).filter((x) => { return x })
-//          list = list.filter((item) => {
-//            return !["caddy", "caddy.exe"].includes(item.name.toLowerCase()) 
-//          })
-//          console.log("#LIST", { list, size: list.length })
           resolve(list)
         });
       });      

package/kernel/prototype.js

@@ -1,5 +1,6 @@
 const fs = require('fs')
 const path = require('path')
+const { glob, sync, hasMagic } = require('glob-gitignore')
 class Proto {
   constructor(kernel) {
     this.kernel = kernel
@@ -10,24 +11,26 @@ class Proto {
     let exists = await this.kernel.exists("prototype")
     if (!exists) {
       console.log("prototype doesn't exist. cloning...")
+      await fs.promises.mkdir(this.kernel.path("prototype"), { recursive: true }).catch((e) => { })
       await this.kernel.exec({
-        message: "git clone https://github.com/peanutcocktail/prototype",
-        path: this.kernel.homedir
+        message: "git clone https://github.com/peanutcocktail/prototype system",
+        path: this.kernel.path("prototype")
       }, (e) => {
         process.stdout.write(e.raw)
       })
     }
-    // get all prototypes
+
+
+    let cwd = path.resolve(this.kernel.homedir, "prototype")
+    let prototype_paths = (await glob('**/pinokio.js', { cwd }))
     let prototype_dir = path.resolve(this.kernel.homedir, "prototype")
-    let prototypes = await fs.promises.readdir(prototype_dir, { withFileTypes: true })
-    prototypes = prototypes.filter(d => d.isDirectory()).map(d => d.name);
-    console.log({ prototypes })
     this.items = []
     this.kv = {}
-    for(let proto of prototypes) {
-      let pinokiojs = path.resolve(prototype_dir, proto, "pinokio.js")
+    for(let prototype_path of prototype_paths) {
+      let proto = path.dirname(prototype_path)
+      let pinokiojs = path.resolve(prototype_dir, prototype_path)
       let config = await this.kernel.require(pinokiojs)
-      if (config && config.type === "prototype") {
+      if (config && config.create && typeof config.create === "function") {
         if (config.icon) {
           config.icon = "/prototype/" + proto + "/" + config.icon
         } else {

package/kernel/router/common.js

@@ -1,50 +1,40 @@
-class Common {
+const Processor = require('./processor')
+class Common extends Processor {
   constructor (router) {
+    super()
     this.router = router
   }
   handle({ match, dial, host }) {
-    this.router.config.apps.http.servers.main.routes.push({
-      "match": [{ "host": [match] }],
-      "handle": [
-        {
-          "handler": "reverse_proxy",
-          "upstreams": [{ "dial": dial }],
-          "headers": {
-            "request": {
-              "set": {
-                "Origin": ["http://127.0.0.1:11434"],
-                "X-Forwarded-Proto": ["https"],
-                "X-Forwarded-Host": ["{http.request.host}"]
-              }
-            },
-            "response": {
-              "set": {
-                "Access-Control-Allow-Origin": ["*"],
-                "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-                "Access-Control-Allow-Headers": ["*"],
-                "Vary": ["Origin"]
-              }
-            }
+    let handler = [{
+      "handler": "reverse_proxy",
+      "upstreams": [{ "dial": dial }],
+      "headers": {
+        "request": {
+          "set": {
+            "X-Forwarded-Proto": ["https"],
+            "X-Forwarded-Host": ["{http.request.host}"]
           }
         },
-        //{
-        //  "handler": "headers",
-        //  "request": {
-        //    "set": {
-        //      "X-Forwarded-Proto": ["https"],
-        //      "X-Forwarded-Host": ["{http.request.host}"]
-        //    }
-        //  },
-        //  "response": {
-        //    "set": {
-        //      "Access-Control-Allow-Origin": ["*"],
-        //      "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-        //      "Access-Control-Allow-Headers": ["*"],
-        //      "Vary": ["Origin"]
-        //    }
-        //  }
-        //}
-      ]
+        "response": {
+          "set": {
+            "Access-Control-Allow-Origin": ["*"],
+            "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+            "Access-Control-Allow-Headers": ["*"],
+            "Vary": ["Origin"]
+          }
+        }
+      }
+    }]
+
+    // if the dial port has been overridden by router.custom_routers, use that instead
+    let parsed_dial = this.parse_ip(dial)
+    let override_handler = this.router.custom_routers[String(parsed_dial.port)]
+    if (override_handler) {
+      handler = override_handler
+    }
+    this.router.config.apps.http.servers.main.routes.push({
+      "match": [{ "host": [match] }],
+      "handle": handler
     })
     this.router.add({ host, dial, match })
   }

package/kernel/router/connector.js

@@ -1,60 +1,48 @@
-class Connector {
+const Processor = require('./processor')
+class Connector extends Processor {
   constructor(router) {
+    super()
     this.router = router
   }
   handle ({ match, connector, dial, }) {
-//    console.log("connector.handle", { match, connector, dial })
     let connector_name = connector.host + ":" + connector.port
     let listener = "0.0.0.0:PORT_PLACEHOLDER_" + connector_name
     let listener_pointer = connector.host + ":PORT_PLACEHOLDER_" + connector_name
     let port_placeholder = "PORT_PLACEHOLDER_" + connector_name
-//    console.log("CONNECTING", { listener, dial, listener_pointer, match })
+
+    let handler = [{
+      "handler": "reverse_proxy",
+      "upstreams": [{ "dial": dial }],
+      "headers": {
+        "request": {
+          "set": {
+            "X-Forwarded-Proto": ["{http.request.header.X-Forwarded-Proto}"],
+            //"X-Forwarded-Proto": ["{http.request.scheme}"],
+            //"X-Forwarded-Proto": ["https"],
+            //"X-Forwarded-Host": ["{http.request.host}"]
+          }
+        },
+        "response": {
+          "set": {
+            "Access-Control-Allow-Origin": ["*"],
+            "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
+            "Access-Control-Allow-Headers": ["*"],
+            "Vary": ["Origin"]
+          }
+        }
+      }
+    }]
+    // if the dial port has been overridden by router.custom_routers, use that instead
+    let parsed_dial = this.parse_ip(dial)
+    let override_handler = this.router.custom_routers[String(parsed_dial.port)]
+    if (override_handler) {
+      handler = override_handler
+    }
     if(!this.router.config.apps.http.servers[connector_name]) {
       this.router.config.apps.http.servers[connector_name] = {
         "listen": [listener],
         "routes": [{
-          "handle": [
-            {
-              "handler": "reverse_proxy",
-              "upstreams": [{ "dial": dial }],
-              "headers": {
-                "request": {
-                  "set": {
-                    "Origin": ["http://127.0.0.1:11434"],
-                    "X-Forwarded-Proto": ["{http.request.header.X-Forwarded-Proto}"],
-                    //"X-Forwarded-Proto": ["{http.request.scheme}"],
-                    //"X-Forwarded-Proto": ["https"],
-                    //"X-Forwarded-Host": ["{http.request.host}"]
-                  }
-                },
-                "response": {
-                  "set": {
-                    "Access-Control-Allow-Origin": ["*"],
-                    "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-                    "Access-Control-Allow-Headers": ["*"],
-                    "Vary": ["Origin"]
-                  }
-                }
-              }
-            },
-//            {
-//              "handler": "headers",
-//              "request": {
-//                "set": {
-//                  "X-Forwarded-Proto": ["https"],
-//                  "X-Forwarded-Host": ["{http.request.host}"]
-//                }
-//              },
-//              "response": {
-//                "set": {
-//                  "Access-Control-Allow-Origin": ["*"],
-//                  "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-//                  "Access-Control-Allow-Headers": ["*"],
-//                  "Vary": ["Origin"]
-//                }
-//              }
-//            }
-          ]
+          "handle": handler
         }]
       }
     }
@@ -85,28 +73,9 @@ class Connector {
             }
           }
         },
-//        {
-//          "handler": "headers",
-//          "request": {
-//            "set": {
-//              "X-Forwarded-Proto": ["https"],
-//              "X-Forwarded-Host": ["{http.request.host}"]
-//            }
-//          },
-//          "response": {
-//            "set": {
-//              "Access-Control-Allow-Origin": ["*"],
-//              "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-//              "Access-Control-Allow-Headers": ["*"],
-//              "Vary": ["Origin"]
-//            }
-//          }
-//        }
       ]
     })
     this.router.add({ host: connector.host, dial: listener_pointer, match })
-//    this.router._tree[connector_name] = dial
-//    this.router._tree[connector_name] = dial
   }
 }
 module.exports = Connector