pinokiod 3.8.147 → 3.8.149

package/kernel/bin/caddy.js

@@ -20,9 +20,11 @@ class Caddy {
     }
   }
   async start() {
+//    console.log(">> proc", this.kernel.processes.caddy_pid, this.kernel.processes.info)
 //    let running = await this.running()
 //    console.log("Running", running)
 //    if (running) {
+//      // kill first
 //      return
 //    } 
     let installed = await this.installed()
@@ -105,12 +107,7 @@ class Caddy {
     console.log("FINISHED")
   }
   async installed() {
-//    let e = true
     let e = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/pki/authorities/local/root.crt"))
-//    let exists = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/certificates/local/pinokio.localhost/pinokio.localhost.crt"))
-//    let exists2 = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/certificates/local/pinokio.localhost/pinokio.localhost.key"))
-//    if (exists && exists2) {
-    console.log("checking installed", e)
     if (e) {
       if (this.kernel.platform === "win32") {
         return this.kernel.bin.installed.conda.has("caddy")
@@ -122,9 +119,13 @@ class Caddy {
     }
   }
   async uninstall(req, ondata) {
+    ondata({ raw: "cleaning up\r\n" })
     await this.kernel.bin.exec({
       message: `conda remove ${this.cmd()}`,
     }, ondata)
+    const folder = this.kernel.path("cache/XDG_DATA_HOME/caddy")
+    await rimraf(folder)
+    ondata({ raw: "finished cleaning up\r\n" })
   }
 }
 module.exports = Caddy

package/kernel/bin/conda.js

@@ -253,8 +253,12 @@ report_errors: false`)
     // 2. The pinned file says conda-libmamba-solver=24.11.1
     // 3. so after conda update --all, it should be conda-libmamba-solver=24.11.1
 
+    console.log({ req })
+
+    
     let mods = this.kernel.bin.mods.filter((m) => {
-      return ["zip", "uv", "node", "huggingface", "gxx", "git", "ffmpeg", "caddy"].includes(m.name)
+      return req.dependencies.includes(m.name)
+//      return ["zip", "uv", "node", "huggingface", "gxx", "git", "ffmpeg", "caddy"].includes(m.name)
     }).map((m) => {
       return m.mod.cmd()
     }).join(" ")

package/kernel/bin/index.js

@@ -28,6 +28,7 @@ const semver = require('semver')
 const Environment = require('../environment')
 //const imageToAscii = require("image-to-ascii");
 
+const Setup = require('./setup')
 
 
 //const Puppet = require("./puppeteer")
@@ -440,33 +441,6 @@ class Bin {
       }
     }
 
-//    // 2. pip
-//    let pip = new Set()
-//    if (conda_exists) {
-//      // conda comes with pip
-//      console.log("## check pip")
-//      start = false
-//      res = await this.exec({ message: `pip list` }, (stream) => {
-////        console.log("pip list check", { stream })
-//      })
-//      console.log("PIP", res.response)
-//      lines = res.response.split(/[\r\n]+/)
-//      for(let line of lines) {
-//        if (start) {
-//          let chunks = line.split(/\s+/).filter(x => x)
-//          if (chunks.length > 1) {
-//            pip.add(chunks[0])
-//          }
-//        } else {
-//          if (/-------.*/i.test(line)) {
-//            start = true 
-//          }
-//        }
-//      }
-//    }
-//    this.installed.pip = pip
-    
-
     if (this.platform === "darwin") {
       // 3. brew
       console.log("## check brew")
@@ -668,7 +642,9 @@ class Bin {
     }
   }
   async install2(req, ondata) {
-    let { requirements, install_required, requirements_pending, error } = await this.check_bin()
+    let { requirements, install_required, requirements_pending, error } = await this.check({
+      bin: this.preset("ai")
+    })
     req.params = JSON.stringify(requirements)
     console.log(">>>>> 1")
     if (this.install_required) {
@@ -678,6 +654,7 @@ class Bin {
     }
   }
   async install(req, ondata) {
+    console.time("kernel.bin.install")
     /*
       req.params := {
         client: {
@@ -778,23 +755,35 @@ class Bin {
       console.log(">>> shell reset")
       this.kernel.shell.reset()
     }
+    console.timeEnd("kernel.bin.install")
     await this.init()
   }
-  async installed(r) {
+  async check_installed(r, dependencies) {
     if (Array.isArray(r.name)) {
       for(let name of r.name) {
-        let installed = await this._installed(name, r.type)
+        let installed = await this._installed(name, r.type, dependencies)
         if (!installed) return false
       }
       return true
     } else {
-      let installed = await this._installed(r.name, r.type)
+      let installed = await this._installed(r.name, r.type, dependencies)
       return installed
     }
   }
-  async _installed(name, type) {
+  async _installed(name, type, dependencies) {
     if (type === "conda") {
-      return this.installed.conda.has(name)
+      let conda_installed = this.installed.conda.has(name)
+      console.log({ dependencies })
+      let dependencies_installed = true
+      for(let d of dependencies) {
+        console.log("checking dependency", d)
+        if(!this.installed.conda.has(d)) {
+          dependencies_installed = false
+          console.log("not installed", d)
+          break
+        }
+      }
+      return conda_installed && dependencies_installed
     } else if (type === "pip") {
       return this.installed.pip && this.installed.pip.has(name)
     } else if (type === "brew") {
@@ -810,50 +799,234 @@ class Bin {
       return installed
     }
   }
-  async check_bin() {
-    let requirements = [
-      { name: "conda", },
-      { name: "git", },
-      { name: "zip", },
-      { name: "node", },
-      { name: "ffmpeg", },
-      { name: "caddy", }
-    ]
+  preset(mode) {
+    return Setup[mode](this.kernel)
+  }
+  requirements(config) {
+    let requirements = config.bin.requirements
+//    let requirements = [
+//      { name: "conda", },
+//      { name: "git", },
+////      { name: "zip", },
+//      { name: "node", },
+//      { name: "ffmpeg", },
+//      { name: "caddy", }
+//    ]
+//    let platform = os.platform()
+//    if (platform === "win32") {
+//      requirements.push({ name: "registry" })
+//      requirements.push({ name: "vs" })
+//    }
+//    if (platform === "darwin") {
+//      requirements.push({ name: "brew" })
+//    }
+//    if (platform === "linux") {
+//      requirements.push({ name: "gxx" })
+//    }
+//    if (this.kernel.gpu === "nvidia") {
+//      requirements.push({ name: "cuda", })
+//    }
+//    requirements = requirements.concat([
+//      { name: "cloudflared" },
+////      { name: "playwright" },
+//      { name: "huggingface" },
+//      { name: "uv" },
+//      { name: "py" },
+//    ])
+    if (config.script && config.script.requires && config.script.requires.length > 0) {
+      /*********************************************************************
+
+      {
+        requires: [{
+          platform,
+          type,
+          name,
+          args
+        }, {
+          platform,
+          type,
+          name,
+          args
+        }],
+        run: [{
+          ...
+        }]
+      }
+
+      syntax :=
+
+        {
+          platform: <win32|darwin|linux>,
+          type: <conda|pip|brew|none>,
+          name: <package name>,           (example: "ffmpeg", "git")
+          args: <install command flags>   (example: "-c conda-forge")
+        }
+
+
+      1. pinokio native install: no need for specifying platforms since they are included
+
+        {
+          name: "conda"
+        }
+
+
+      2. non native install (conda, pip, brew)
+
+        2.1. Same on all platforms 
+
+        [{
+          type: "conda",
+          name: "ffmpeg",
+          args: "-c conda-forge"
+        }]
+
+        2.2. Specify per platform
+
+
+        [
+          { name: "conda" },
+          { platform: "darwin", type: "brew", name: "llvm" },
+          { platform: "linux", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" },
+          { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
+        ]
+
+        [
+          { name: "conda" },
+          { platform: ["darwin", "linux"], type: "brew", name: "llvm" },
+          { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
+        ]
+
+
+
+      *********************************************************************/
+      let type_name_set = new Set()
+      for(let r of config.script.requires) {
+        // if no platform specified, or if the specified platform matches the current platform
+        if (!r.platform || platform === r.platform || Array.isArray(r.platform) && r.platform.includes(platform) ) {
+          if (Array.isArray(r.name)) {
+            // if array, just add it
+            requirements.push(r)
+          } else {
+            let type_name = `${r.type ? r.type : ''}/${r.name}`
+            if (!type_name_set.has(type_name)) {
+              type_name_set.add(type_name)
+              requirements.push(r)
+            }
+          }
+        }
+      }
+
+    }
+    return requirements
+  }
+  relevant(r) {
+    /*
+      single platform
+      [
+        { name: "conda" },
+        { platform: "darwin", type: "brew", name: "llvm" },
+        { platform: "linux", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" },
+        { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
+      ]
+
+      multiple platforms
+      [
+        { name: "conda" },
+        { platform: ["darwin", "linux"], type: "brew", name: "llvm" },
+        { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
+      ]
+
+
+      platform & arch
+      [
+        { name: "conda" },
+        { platform: "darwin", arch: ["arm64", "x64"], type: "brew", name: "llvm" },
+      ]
+    */
+
     let platform = os.platform()
-    if (platform === "win32") {
-      requirements.push({ name: "registry" })
-      requirements.push({ name: "vs" })
+    let arch = os.arch()
+    let gpu = this.kernel.gpu
+    let relevant = {
+      platform: false,
+      arch: false,
+      gpu: false,
     }
-    if (platform === "darwin") {
-      requirements.push({ name: "brew" })
+    if (r.platform) {
+      if (Array.isArray(r.platform)) {
+        // multiple items
+        if (r.platform.includes(platform)) {
+          relevant.platform = true
+        }
+      } else {
+        // one item
+        if (r.platform === platform) {
+          relevant.platform = true
+        }
+      }
+    } else {
+      // all platforms
+      relevant.platform = true
     }
-    if (platform === "linux") {
-      requirements.push({ name: "gxx" })
+    if (r.arch) {
+      if (Array.isArray(r.arch)) {
+        // multiple items
+        if (r.arch.includes(arch)) {
+          relevant.arch = true
+        }
+      } else {
+        // one item
+        if (r.arch === arch) {
+          relevant.arch = true
+        }
+      }
+    } else {
+      // all platforms
+      relevant.arch = true
     }
-    if (this.kernel.gpu === "nvidia") {
-      requirements.push({ name: "cuda", })
+    if (r.gpu) {
+      if (Array.isArray(r.gpu)) {
+        // multiple items
+        if (r.gpu.includes(gpu)) {
+          relevant.gpu = true
+        }
+      } else {
+        // one item
+        if (r.gpu === gpu) {
+          relevant.gpu = true
+        }
+      }
+    } else {
+      // all platforms
+      relevant.gpu = true
     }
-    requirements = requirements.concat([
-      { name: "cloudflared" },
-//      { name: "playwright" },
-      { name: "huggingface" },
-      { name: "uv" },
-      { name: "py" },
-    ])
-
+    return relevant.platform && relevant.arch && relevant.gpu
+  }
+  async check(config) {
+    let requirements = this.requirements(config)
     let requirements_pending = !this.installed_initialized
     let install_required = true
     if (!requirements_pending) {
       install_required = false
       for(let i=0; i<requirements.length; i++) {
         let r = requirements[i]
-        console.time(r.name)
-        let installed = await this.installed(r)
-        console.log({ r, installed })
-        console.timeEnd(r.name)
-        requirements[i].installed = installed
-        if (!installed) {
-          install_required = true
+        //let installed = await this.installed(r)
+        //requirements[i].installed = installed
+        //if (!installed) {
+        //  install_required = true
+        //}
+        let relevant = this.relevant(r)
+        requirements[i].relevant = relevant
+        if (relevant) {
+          let dependencies
+          if (r.name === "conda") {
+            dependencies = config.bin.conda_requirements
+          }
+          let installed = await this.check_installed(r, dependencies)
+          requirements[i].installed = installed
+          if (!installed) {
+            install_required = true
+          }
         }
       }
     }
@@ -870,6 +1043,11 @@ class Bin {
     this.requirements_pending = requirements_pending
 
     console.log("check_bin finished 2")
+
+    requirements = requirements.filter((r) => {
+      return r.relevant
+    })
+
     return {
       error,
       requirements,

package/kernel/bin/setup.js

@@ -0,0 +1,130 @@
+const os = require('os')
+let platform = os.platform()
+module.exports = {
+  ai: (kernel) => {
+    let conda_requirements = [
+      "uv",
+      "node",
+      "huggingface",
+      "git",
+      "ffmpeg",
+//      "caddy"
+    ]
+    let requirements = [
+      { name: "conda", },
+      { name: "git", },
+  //      { name: "zip", },
+      { name: "node", },
+      { name: "ffmpeg", },
+//      { name: "caddy", }
+    ]
+    if (platform === "win32") {
+      requirements.push({ name: "registry" })
+      requirements.push({ name: "vs" })
+    }
+    if (platform === "darwin") {
+      requirements.push({ name: "brew" })
+    }
+    if (platform === "linux") {
+      requirements.push({ name: "gxx" })
+      conda_requirements.push("gxx")
+    }
+    if (kernel.gpu === "nvidia") {
+      requirements.push({ name: "cuda", })
+      conda_requirements.push("cuda")
+    }
+    requirements = requirements.concat([
+      { name: "cloudflared" },
+//      { name: "playwright" },
+      { name: "huggingface" },
+      { name: "uv" },
+      { name: "py" },
+    ])
+    return {
+      icon: "fa-solid fa-brain",
+      title: "AI",
+      description: "Install common modules required for running AI locally",
+      requirements,
+      conda_requirements
+    }
+  },
+  javascript: (kernel) => {
+    return {
+      icon: "fa-brands fa-js",
+      title: "Node.js",
+      description: "Set up self-contained node.js development environment",
+      requirements: [
+        { name: "conda", },
+        { name: "git", },
+        { name: "node", },
+      ],
+      conda_requirements: [
+        "node",
+        "git",
+      ]
+    }
+  },
+  python: (kernel) => {
+    return {
+      icon: "fa-brands fa-python",
+      title: "Python",
+      description: "Set up self-contained python development environment",
+      requirements: [
+        { name: "conda", },
+        { name: "git", },
+        { name: "uv", },
+      ],
+      conda_requirements: [
+        "uv",
+        "git",
+      ]
+    }
+  },
+  dev: (kernel) => {
+    let requirements = [
+      { name: "conda", },
+      { name: "git", },
+      { name: "node", },
+      { name: "uv", },
+    ]
+    let conda_requirements = [
+      "uv",
+      "node",
+      "git",
+    ]
+    if (platform === "win32") {
+      requirements.push({ name: "registry" })
+      requirements.push({ name: "vs" })
+    }
+    if (platform === "darwin") {
+      requirements.push({ name: "brew" })
+    }
+    if (platform === "linux") {
+      requirements.push({ name: "gxx" })
+      conda_requirements.push("gxx")
+    }
+    return {
+      icon: "fa-solid fa-laptop-code",
+      title: "Coding",
+      description: "Install common modules required for development (Node.js, python, Visual Studio Developer Tools (Windows), Xcode build tools (Mac)",
+      requirements,
+      conda_requirements,
+    }
+  },
+  network: (kernel) => {
+    return {
+      icon: "fa-solid fa-wifi",
+      title: "Network",
+      description: "Automatic HTTPS and local network sharing for ALL local apps",
+      requirements: [
+        { name: "conda", },
+        { name: "git", },
+        { name: "caddy", },
+      ],
+      conda_requirements: [
+        "git",
+        "caddy"
+      ]
+    }
+  }
+}

package/kernel/environment.js

@@ -439,4 +439,39 @@ const get_raw = async (homedir) => {
   }
   return current_env
 }
-module.exports = { ENV, get, get2, init_folders }
+
+const requirements = async (script, filepath, kernel) => {
+  let pre_items = []
+  let requires_instantiation = false
+  if (script && script.pre) {
+    let env = await get2(filepath, kernel)
+    for(let item of script.pre) {
+      if (item.env) {
+
+        if (!item.index) {
+          item.index = 0
+        }
+        if (item.key) {
+          const hasProtocol = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(item.key);
+          const url = new URL(hasProtocol ? item.key : `https://${item.key}`);
+          item.host = url.host
+          item.val = await kernel.kv.get(item.host, item.index)
+        } else {
+          item.host = ""
+        }
+
+        if (env[item.env]) {
+          item.val = env[item.env]
+        } else {
+          if (item.default) {
+            item.val = item.default
+          }
+          requires_instantiation = true
+        }
+        pre_items.push(item)
+      }
+    }
+  }
+  return { items: pre_items, requires_instantiation }
+}
+module.exports = { ENV, get, get2, init_folders, requirements }

package/kernel/index.js

@@ -204,7 +204,6 @@ class Kernel {
     if (this.router.port_mapping && Object.keys(this.router.port_mapping).length > 0) {
       let max_caddy_port = Math.max(...Object.values(this.router.port_mapping))
       let start_port = Math.max(42003, max_caddy_port + 1)
-      console.log({ start_port })
       return start_port
     } else {
       return 42003
@@ -267,36 +266,50 @@ class Kernel {
     if (env && env.PINOKIO_NETWORK_ACTIVE && (env.PINOKIO_NETWORK_ACTIVE==="1" || env.PINOKIO_NETWORK_ACTIVE.toLowerCase()==="true")) {
       active = true
     }
-    console.log("kernel.refresh", { active, notify_peers })
+//    console.log("kernel.refresh", { active, notify_peers })
     if (active) {
 
       // 1. get the process list
-      console.log("Step 1: Get process list")
+//      console.log("Step 1: Get process list")
+      console.log("> 1")
       await this.processes.refresh()
-      console.log("Process info", this.processes.info)
+//      console.log("Process info", this.processes.info)
+
+      console.log("> 2")
 
       // 2. refresh peer info to reflect the proc info
       await this.peer.refresh()
 
+
+      console.log("> 3")
+      // 3. load custom routers from ~/pinokio/network
+      await this.router.init()
+
+      console.log("> 4")
       // 3. set current local host router info
-      console.log("Step 3: Set current local host router")
+//      console.log("Step 3: Set current local host router")
       await this.router.local()
-      console.log("router mapping after step 3", this.router.info)
+//      console.log("router mapping after step 3", this.router.info)
 
+      console.log("> 5")
       // 4. refresh peer info to reflect the updated router info
       await this.peer.refresh()
 
+      console.log("> 6")
       // 4. tell peers to refresh
       if (notify_peers) {
-        console.log("Step 4: notify peers to refresh")
+//        console.log("Step 4: notify peers to refresh")
+      console.log("> 7")
         await this.peer.notify_peers()
       }
 
+      console.log("> 8")
       // 4. update remote router
-      console.log("Step 5: Set remote routers")
+//      console.log("Step 5: Set remote routers")
       await this.router.remote()
-      console.log("router mapping after step 5", this.router.info)
+//      console.log("router mapping after step 5", this.router.info)
 
+      console.log("> 9")
       await this.router.update()
 
 
@@ -669,10 +682,8 @@ class Kernel {
       this.bin.init().then(() => {
         if (this.homedir) {
           this.shell.init().then(async () => {
-            this.bin.check_bin()
-            console.log(">>>> 1")
+//            this.bin.check()
             if (this.envs) {
-              console.log(">>>> 2")
               this.template.update({
                 env: this.envs,
                 envs: this.envs,
@@ -721,7 +732,6 @@ class Kernel {
                         }
                       }
                     }
-                console.log(">>>>>> d")
                     clearInterval(interval)
                     setTimeout(() => {
                       this.launch_complete = true