pinokiod 3.8.149 → 3.8.150

package/kernel/api/index.js

@@ -119,10 +119,8 @@ class Api {
     return result
   }
   async stop(req, ondata) {
-    console.log(">>>>>>>>>>>>>>>>>> STOP")
     // 1. set the "stop" flag for the uri, so the next execution in the queue for the uri will NOT queue another task
     // 2. stream a message closing the socket
-    console.time("stop 1")
 
     let requestPath = this.filePath(req.params.uri)
 
@@ -132,9 +130,6 @@ class Api {
         await this.process(script.on.stop) 
       }
     }
-    console.timeEnd("stop 1")
-
-    console.time("stop 2")
     // reset modules
     let modpath = this.resolvePath(cwd, req.params.uri)
     console.log("STOP modpath", modpath)
@@ -160,15 +155,11 @@ class Api {
 
     // stop all proxies
     this.stopProxy({ script: requestPath })
-    console.timeEnd("stop 2")
 
     // stop all cloudflare tunnels
 
-    console.time("stop 3")
     await this.kernel.stopCloudflare({ path: requestPath })
-    console.timeEnd("stop 3")
 
-    console.time("stop 4")
 
     // if there are any pending waiters, delete them
 
@@ -185,7 +176,6 @@ class Api {
       id: requestPath,
       type: "disconnect"
     })
-    console.timeEnd("stop 4")
     this.kernel.refresh(true)
     return true
   }

package/kernel/api/script/index.js

@@ -1,7 +1,6 @@
 const path = require('path')
 class Script {
   async start(req, ondata, kernel) {
-    console.log(">>>>>>>>> script.start", req)
     let res = await this.run(req, ondata, kernel)
     return res
   }
@@ -44,7 +43,6 @@ class Script {
     */
     // if already running, print that it's already running
     let id = kernel.api.filePath(req.params.uri, req.cwd)
-    console.log({ id , uri: req.params.uri, cwd: req.cwd })
     if (kernel.api.running[id]) {
       let msg = `${req.params.uri} already running. Continuing...\r\n`
       ondata({ raw: msg })

package/kernel/bin/caddy.js

@@ -79,7 +79,6 @@ class Caddy {
         ondata(e)
       })
     } else {
-      console.log("req", req)
       ondata({
         title: "Password",
         description: "Enter your system password to add the HTTPS certificate",
@@ -90,9 +89,7 @@ class Caddy {
           placeholder: "System password",
         }]
       }, "input")
-      console.log("Wait until", id)
       let response = await this.kernel.api.wait(id)
-      console.log({ response })
       await this.kernel.exec({
         message: `echo ${response.password} | sudo -S caddy trust`,
         path: this.kernel.homedir
@@ -104,7 +101,6 @@ class Caddy {
   //        }
       })
     }
-    console.log("FINISHED")
   }
   async installed() {
     let e = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/pki/authorities/local/root.crt"))

package/kernel/bin/conda.js

@@ -120,10 +120,9 @@ report_errors: false`)
 //  }
   async check() {
     let res = await this.kernel.bin.exec({ message: `conda list` }, (stream) => {
-      process.stdout.write(stream.raw)
+//      process.stdout.write(stream.raw)
 //        console.log("conda list check", { stream })
     })
-    console.log("2 conda list check", res)
 
     let lines = res.response.split(/[\r\n]+/)
     let conda_check = {}
@@ -179,7 +178,6 @@ report_errors: false`)
         }
       }
     }
-    console.log("> Check", { conda_check, conda })
     this.kernel.bin.installed.conda = conda
     return conda_check.conda && conda_check.mamba && conda_check.sqlite
     //return conda_check.conda && conda_check.mamba
@@ -253,21 +251,17 @@ report_errors: false`)
     // 2. The pinned file says conda-libmamba-solver=24.11.1
     // 3. so after conda update --all, it should be conda-libmamba-solver=24.11.1
 
-    console.log({ req })
-
-    
     let mods = this.kernel.bin.mods.filter((m) => {
       return req.dependencies.includes(m.name)
 //      return ["zip", "uv", "node", "huggingface", "gxx", "git", "ffmpeg", "caddy"].includes(m.name)
     }).map((m) => {
       return m.mod.cmd()
     }).join(" ")
-    console.log("To install", { mods })
+    console.log("Conda dependencies to install", { mods })
 
     let cmds = [
       //"conda clean -y --index-cache",
       "conda clean -y --all",
-      //"conda install -y -c conda-forge sqlite=3.47.2",
       `conda install -y -c conda-forge sqlite=3.47.2 ${mods}`,
 
 //      `conda config --file ${this.kernel.path('condarc')} --set remote_connect_timeout_secs 20`,
@@ -381,9 +375,6 @@ report_errors: false`)
     let e
     for(let p of this.paths[this.kernel.platform]) {
       let e = await this.kernel.bin.exists(p)
-      console.log("kernel.bin.conda.installed")
-      console.log("e", e)
-      console.log("correct_conda", this.kernel.bin.correct_conda)
       if (e && this.kernel.bin.correct_conda) return true
     }
     return false

package/kernel/bin/index.js

@@ -345,11 +345,8 @@ class Bin {
       this.correct_conda = false
     } else {
       res = await this.exec({ message: `conda list` }, (stream) => {
-        process.stdout.write(stream.raw)
   //        console.log("conda list check", { stream })
       })
-      console.log("1 conda list check", res)
-
 
       let lines = res.response.split(/[\r\n]+/)
       for(let line of lines) {
@@ -402,7 +399,6 @@ class Bin {
           }
         }
       }
-      console.log("conda_check", conda_check)
 
       if (conda_check.conda && conda_check.mamba && conda_check.sqlite) {
       //if (conda_check.conda && conda_check.mamba) {
@@ -646,15 +642,12 @@ class Bin {
       bin: this.preset("ai")
     })
     req.params = JSON.stringify(requirements)
-    console.log(">>>>> 1")
     if (this.install_required) {
-      console.log(">>>>> 2")
       let res = await this.install(req, ondata)
       return res
     }
   }
   async install(req, ondata) {
-    console.time("kernel.bin.install")
     /*
       req.params := {
         client: {
@@ -672,13 +665,7 @@ class Bin {
     } else {
       this.client = null
     }
-
-    console.log("install", req)
-
     let requirements = JSON.parse(req.params)
-
-    console.time("### Total Install")
-
     for(let x=0; x<10; x++) {
       console.log(`## Install Attempt ${x}`)
       let i = 0;
@@ -692,9 +679,7 @@ class Bin {
           console.log("already installed. skip.")
         } else {
           console.log("not installed. install.")
-          console.time("############# Install " +requirement.name)
           await this.tryInstall(req.id, requirement, x, i, requirements.length, ondata)
-          console.timeEnd("############# Install " +requirement.name)
         }
       }
 
@@ -750,12 +735,10 @@ class Bin {
     //    }
     //  }
     //}
-    console.timeEnd("### Total Install")
     if (this.kernel.shell) {
       console.log(">>> shell reset")
       this.kernel.shell.reset()
     }
-    console.timeEnd("kernel.bin.install")
     await this.init()
   }
   async check_installed(r, dependencies) {
@@ -804,35 +787,6 @@ class Bin {
   }
   requirements(config) {
     let requirements = config.bin.requirements
-//    let requirements = [
-//      { name: "conda", },
-//      { name: "git", },
-////      { name: "zip", },
-//      { name: "node", },
-//      { name: "ffmpeg", },
-//      { name: "caddy", }
-//    ]
-//    let platform = os.platform()
-//    if (platform === "win32") {
-//      requirements.push({ name: "registry" })
-//      requirements.push({ name: "vs" })
-//    }
-//    if (platform === "darwin") {
-//      requirements.push({ name: "brew" })
-//    }
-//    if (platform === "linux") {
-//      requirements.push({ name: "gxx" })
-//    }
-//    if (this.kernel.gpu === "nvidia") {
-//      requirements.push({ name: "cuda", })
-//    }
-//    requirements = requirements.concat([
-//      { name: "cloudflared" },
-////      { name: "playwright" },
-//      { name: "huggingface" },
-//      { name: "uv" },
-//      { name: "py" },
-//    ])
     if (config.script && config.script.requires && config.script.requires.length > 0) {
       /*********************************************************************
 

package/kernel/environment.js

@@ -447,7 +447,7 @@ const requirements = async (script, filepath, kernel) => {
     let env = await get2(filepath, kernel)
     for(let item of script.pre) {
       if (item.env) {
-
+        // if index is not set, use 0 as default (for key)
         if (!item.index) {
           item.index = 0
         }
@@ -464,7 +464,21 @@ const requirements = async (script, filepath, kernel) => {
           item.val = env[item.env]
         } else {
           if (item.default) {
-            item.val = item.default
+            if (typeof item.default === 'object') {
+              if (item.default.key) {
+                const hasProtocol = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(item.default.key);
+                const url = new URL(hasProtocol ? item.default.key : `https://${item.default.key}`);
+                item.host = url.host
+
+                let index = item.default.index || 0
+                item.index = index
+                item.val = await kernel.kv.get(item.host, index)
+              } else {
+                item.host = ""
+              }
+            } else {
+              item.val = item.default
+            }
           }
           requires_instantiation = true
         }

package/kernel/index.js

@@ -118,21 +118,6 @@ class Kernel {
       }
     }
   }
-//  path_exists(...args) {
-//    return this.info.exists(...args)
-//  }
-//  path_resolve(...args) {
-//    return this.info.path(...args)
-//  }
-//  script_running(...args) {
-//    return this.info.running(...args)
-//  }
-//  script_local(...args) {
-//    return this.info.local(...args)
-//  }
-//  script_global(...args) {
-//    return this.info.global(...args)
-//  }
   async env(...args) {
     let folderpath
     if (args) {
@@ -267,69 +252,40 @@ class Kernel {
       active = true
     }
 //    console.log("kernel.refresh", { active, notify_peers })
-    if (active) {
+
+    let installed = await this.bin.check_installed({ name: "caddy" })
+    console.log("caddy installed?", installed)
+    if (installed && active) {
 
       // 1. get the process list
-//      console.log("Step 1: Get process list")
-      console.log("> 1")
       await this.processes.refresh()
-//      console.log("Process info", this.processes.info)
-
-      console.log("> 2")
 
       // 2. refresh peer info to reflect the proc info
       await this.peer.refresh()
 
-
-      console.log("> 3")
       // 3. load custom routers from ~/pinokio/network
       await this.router.init()
 
-      console.log("> 4")
-      // 3. set current local host router info
-//      console.log("Step 3: Set current local host router")
+      // 4. set current local host router info
       await this.router.local()
-//      console.log("router mapping after step 3", this.router.info)
 
-      console.log("> 5")
-      // 4. refresh peer info to reflect the updated router info
+      // 5. refresh peer info to reflect the updated router info
       await this.peer.refresh()
 
-      console.log("> 6")
-      // 4. tell peers to refresh
+      // 6. tell peers to refresh
       if (notify_peers) {
-//        console.log("Step 4: notify peers to refresh")
-      console.log("> 7")
         await this.peer.notify_peers()
       }
 
-      console.log("> 8")
-      // 4. update remote router
-//      console.log("Step 5: Set remote routers")
+      // 7. update remote router
       await this.router.remote()
-//      console.log("router mapping after step 5", this.router.info)
 
-      console.log("> 9")
+      // 8. update caddy config
       await this.router.update()
 
+      // 9. announce self to the peer network
+      this.peer.announce()
 
-//      await Promise.all([
-//        // refresh proc info
-//        this.processes.refresh(),
-//        // refresh peer info
-//        this.peer.refresh()
-//      ])
-//      console.log("proc info", this.processes.info)
-//      console.log("peer info", this.peer.info)
-//      // refresh router info
-//      await this.router.refresh()
-//      console.log("router info", this.router.published())
-////      await this.peer.refresh()
-//
-//      if (notify_peers) {
-//        // broadcast to peers to refresh
-//        await this.peer.notify_peers()
-//      }
     }
   }
   async clearLog(group) {
@@ -542,25 +498,13 @@ class Kernel {
     let home = this.store.get("home")
 
     // reset shells if they exist
-    console.log("KERNEL INIT")
     if (this.shell) {
-      console.log("> shell reset")
       this.shell.reset()
     }
     if (this.peer) {
-      console.log("> peer stop")
       this.peer.stop()
     }
 
-//    // kill all processes just in case
-//    await new Promise((resolve, reject) => {
-//      console.log("killing child processes for", process.pid)
-//      kill(process.pid, {}, 'SIGKILL', () => {
-//        console.log("child procs killed for", process.pid)
-//        resolve()
-//      });
-//    })
-
     this.info = new Info(this)
     this.pipe = new Pipe(this)
     this.proto = new Proto(this)

package/kernel/info.js

@@ -40,16 +40,13 @@ class Info {
   running(...args) {
     let cwd = path.dirname(this.caller())
     let resolved_path = path.resolve(cwd, ...args)
-    console.log("api.running", this.kernel.api.running)
     let running = this.kernel.api.running[resolved_path]
-    console.log("info.running", { cwd, resolved_path, running })
     return running ? running : false
   }
   exists(...args) {
     let cwd = path.dirname(this.caller())
     let resolved_path = path.resolve(cwd, ...args)
     let exists = fs.existsSync(resolved_path)
-    console.log("info.exists", { cwd, resolved_path, exists })
     return exists ? exists: false
   }
   local(...args) {

package/kernel/kv.js

@@ -12,6 +12,9 @@ class KV {
         json[host] = []
       }
       // if index doesn't exist, set index 0
+      if (!index) {
+        index = 0
+      }
       json[host][index] = val
       await fs.promises.writeFile(filePath, JSON.stringify(json, null, 2), 'utf8')
     }

package/kernel/loader.js

@@ -6,14 +6,6 @@ const clearModule = require('clear-module');
 
 class Loader {
   async load(_path) {
-
-//    let exists = await new Promise(r=>fs.access(_path, fs.constants.F_OK, e => r(!e)))
-//    if (!exists) {
-//      console.log(`[load] ${_path} does not exist`)
-//      return { resolved: null }
-//    }
-
-
     let resolved
     let dirname
     let extension = path.extname(_path)
@@ -51,21 +43,18 @@ class Loader {
   async requireJSON(filepath) {
     let config
     try { config = require(filepath) } catch (e) {
-//      console.log(`[did not load] ${filepath}`)
     }
     return config
   }
   async requireJS(filepath) {
     let config
     try { config = require(filepath) } catch (e) {
-//      console.log(`[did not load] ${filepath}`)
     }
     try {
       // if the required module is a class, return the instantiated object
       return new config();
     } catch (e) {
       // otherwise return normally
-//      console.log(`[did not load] ${filepath}`)
       return config
     }
   }

package/kernel/peer.js

@@ -10,6 +10,7 @@ class PeerDiscovery {
     this.peers = new Set();
     this.host = this._getLocalIPAddress()
     this.default_port = 42000
+    this.peers.add(this.host)
 //    this.start();
   }
   stop() {
@@ -19,6 +20,9 @@ class PeerDiscovery {
       this.socket.close()
     }
   }
+  announce() {
+    this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
+  }
   async start(kernel) {
     let env = await Environment.get(kernel.homedir)
 
@@ -36,12 +40,9 @@ class PeerDiscovery {
 
     if (this.active) {
       // Listen for incoming pings
-      console.log("kernel.peer start")
       this.socket = dgram.createSocket('udp4');
       this.socket.on('message', (msg, rinfo) => {
-//        console.log("peer message", { msg, rinfo })
         const ip = rinfo.address;
-//        console.log({rinfo})
         if (msg.toString() === this.message.toString() && this._isLocalLAN(ip)) {
           if (!this.peers.has(ip)) {
             console.log(`Discovered peer: ${ip}`);
@@ -62,16 +63,13 @@ class PeerDiscovery {
 
         this.socket.setBroadcast(true);
 
-        // Send broadcast pings every interval
-        this.interval_handle = setInterval(() => {
-//          console.log("send message to peers")
-          //this.socket.send(this.message, 0, this.message.length, this.port, '127.0.0.1');
-          //console.log("sent to 127.0.0.1")
-          //this.socket.send(this.message, 0, this.message.length, this.port, '255.255.255.255');
-          this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
-          //console.log("sent to 255.255.255.255")
-//          console.log("sent message")
-        }, this.interval);
+        this.announce()
+
+//        // Send broadcast pings every interval
+//        this.interval_handle = setInterval(() => {
+//          //this.socket.send(this.message, 0, this.message.length, this.port, '255.255.255.255');
+//          this.socket.send(this.message, 0, this.message.length, this.port, '192.168.1.255');
+//        }, this.interval);
       });
     }
   }
@@ -86,7 +84,6 @@ class PeerDiscovery {
     }
   }
   async _broadcast(host) {
-//    console.log("peer _broadcast", host)
     try {
       let res = await axios.post(`http://${host}:${this.default_port}/pinokio/peer/refresh`, {
         timeout: 2000
@@ -119,7 +116,6 @@ class PeerDiscovery {
       let peer_info = await Promise.all(refresh_peers.map((host) => {
         return this._refresh(host)
       }))
-//      console.log({ peer_info })
       for(let i=0; i<peer_info.length; i++) {
         let peer = peer_info[i]
         if (peer) {
@@ -129,7 +125,6 @@ class PeerDiscovery {
           }
         }
       }
-//      console.log("REFRESH", this.info)
       this.refreshing = false
       return this.info
     }

package/kernel/procs.js

@@ -105,10 +105,6 @@ class Procs {
               return { port, pid , name, fullname, ip }
             }
           }).filter((x) => { return x })
-//          list = list.filter((item) => {
-//            return !["caddy", "caddy.exe"].includes(item.name.toLowerCase()) 
-//          })
-//          console.log("#LIST", { list, size: list.length })
           resolve(list)
         });
       });      

package/kernel/router/common.js

@@ -11,7 +11,6 @@ class Common extends Processor {
       "headers": {
         "request": {
           "set": {
-//            "Origin": ["http://127.0.0.1:11434"],
             "X-Forwarded-Proto": ["https"],
             "X-Forwarded-Host": ["{http.request.host}"]
           }
@@ -25,25 +24,7 @@ class Common extends Processor {
           }
         }
       }
-    },
-    //{
-    //  "handler": "headers",
-    //  "request": {
-    //    "set": {
-    //      "X-Forwarded-Proto": ["https"],
-    //      "X-Forwarded-Host": ["{http.request.host}"]
-    //    }
-    //  },
-    //  "response": {
-    //    "set": {
-    //      "Access-Control-Allow-Origin": ["*"],
-    //      "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-    //      "Access-Control-Allow-Headers": ["*"],
-    //      "Vary": ["Origin"]
-    //    }
-    //  }
-    //}
-    ]
+    }]
 
     // if the dial port has been overridden by router.custom_routers, use that instead
     let parsed_dial = this.parse_ip(dial)

package/kernel/router/connector.js

@@ -5,12 +5,10 @@ class Connector extends Processor {
     this.router = router
   }
   handle ({ match, connector, dial, }) {
-//    console.log("connector.handle", { match, connector, dial })
     let connector_name = connector.host + ":" + connector.port
     let listener = "0.0.0.0:PORT_PLACEHOLDER_" + connector_name
     let listener_pointer = connector.host + ":PORT_PLACEHOLDER_" + connector_name
     let port_placeholder = "PORT_PLACEHOLDER_" + connector_name
-//    console.log("CONNECTING", { listener, dial, listener_pointer, match })
 
     let handler = [{
       "handler": "reverse_proxy",
@@ -18,7 +16,6 @@ class Connector extends Processor {
       "headers": {
         "request": {
           "set": {
-//            "Origin": ["http://127.0.0.1:11434"],
             "X-Forwarded-Proto": ["{http.request.header.X-Forwarded-Proto}"],
             //"X-Forwarded-Proto": ["{http.request.scheme}"],
             //"X-Forwarded-Proto": ["https"],
@@ -34,23 +31,6 @@ class Connector extends Processor {
           }
         }
       }
-//            {
-//              "handler": "headers",
-//              "request": {
-//                "set": {
-//                  "X-Forwarded-Proto": ["https"],
-//                  "X-Forwarded-Host": ["{http.request.host}"]
-//                }
-//              },
-//              "response": {
-//                "set": {
-//                  "Access-Control-Allow-Origin": ["*"],
-//                  "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-//                  "Access-Control-Allow-Headers": ["*"],
-//                  "Vary": ["Origin"]
-//                }
-//              }
-//            }
     }]
     // if the dial port has been overridden by router.custom_routers, use that instead
     let parsed_dial = this.parse_ip(dial)
@@ -93,28 +73,9 @@ class Connector extends Processor {
             }
           }
         },
-//        {
-//          "handler": "headers",
-//          "request": {
-//            "set": {
-//              "X-Forwarded-Proto": ["https"],
-//              "X-Forwarded-Host": ["{http.request.host}"]
-//            }
-//          },
-//          "response": {
-//            "set": {
-//              "Access-Control-Allow-Origin": ["*"],
-//              "Access-Control-Allow-Methods": ["GET, POST, OPTIONS, PUT, DELETE"],
-//              "Access-Control-Allow-Headers": ["*"],
-//              "Vary": ["Origin"]
-//            }
-//          }
-//        }
       ]
     })
     this.router.add({ host: connector.host, dial: listener_pointer, match })
-//    this.router._tree[connector_name] = dial
-//    this.router._tree[connector_name] = dial
   }
 }
 module.exports = Connector