pinokiod 3.8.148 → 3.8.150

package/kernel/api/index.js

@@ -119,10 +119,8 @@ class Api {
     return result
   }
   async stop(req, ondata) {
-    console.log(">>>>>>>>>>>>>>>>>> STOP")
     // 1. set the "stop" flag for the uri, so the next execution in the queue for the uri will NOT queue another task
     // 2. stream a message closing the socket
-    console.time("stop 1")
 
     let requestPath = this.filePath(req.params.uri)
 
@@ -132,9 +130,6 @@ class Api {
         await this.process(script.on.stop) 
       }
     }
-    console.timeEnd("stop 1")
-
-    console.time("stop 2")
     // reset modules
     let modpath = this.resolvePath(cwd, req.params.uri)
     console.log("STOP modpath", modpath)
@@ -160,15 +155,11 @@ class Api {
 
     // stop all proxies
     this.stopProxy({ script: requestPath })
-    console.timeEnd("stop 2")
 
     // stop all cloudflare tunnels
 
-    console.time("stop 3")
     await this.kernel.stopCloudflare({ path: requestPath })
-    console.timeEnd("stop 3")
 
-    console.time("stop 4")
 
     // if there are any pending waiters, delete them
 
@@ -185,7 +176,6 @@ class Api {
       id: requestPath,
       type: "disconnect"
     })
-    console.timeEnd("stop 4")
     this.kernel.refresh(true)
     return true
   }

package/kernel/api/script/index.js

@@ -1,7 +1,6 @@
 const path = require('path')
 class Script {
   async start(req, ondata, kernel) {
-    console.log(">>>>>>>>> script.start", req)
     let res = await this.run(req, ondata, kernel)
     return res
   }
@@ -44,7 +43,6 @@ class Script {
     */
     // if already running, print that it's already running
     let id = kernel.api.filePath(req.params.uri, req.cwd)
-    console.log({ id , uri: req.params.uri, cwd: req.cwd })
     if (kernel.api.running[id]) {
       let msg = `${req.params.uri} already running. Continuing...\r\n`
       ondata({ raw: msg })

package/kernel/bin/caddy.js

@@ -20,9 +20,11 @@ class Caddy {
     }
   }
   async start() {
+//    console.log(">> proc", this.kernel.processes.caddy_pid, this.kernel.processes.info)
 //    let running = await this.running()
 //    console.log("Running", running)
 //    if (running) {
+//      // kill first
 //      return
 //    } 
     let installed = await this.installed()
@@ -77,7 +79,6 @@ class Caddy {
         ondata(e)
       })
     } else {
-      console.log("req", req)
       ondata({
         title: "Password",
         description: "Enter your system password to add the HTTPS certificate",
@@ -88,9 +89,7 @@ class Caddy {
           placeholder: "System password",
         }]
       }, "input")
-      console.log("Wait until", id)
       let response = await this.kernel.api.wait(id)
-      console.log({ response })
       await this.kernel.exec({
         message: `echo ${response.password} | sudo -S caddy trust`,
         path: this.kernel.homedir
@@ -102,15 +101,9 @@ class Caddy {
   //        }
       })
     }
-    console.log("FINISHED")
   }
   async installed() {
-//    let e = true
     let e = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/pki/authorities/local/root.crt"))
-//    let exists = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/certificates/local/pinokio.localhost/pinokio.localhost.crt"))
-//    let exists2 = await this.kernel.exists(this.kernel.path("cache/XDG_DATA_HOME/caddy/certificates/local/pinokio.localhost/pinokio.localhost.key"))
-//    if (exists && exists2) {
-    console.log("checking installed", e)
     if (e) {
       if (this.kernel.platform === "win32") {
         return this.kernel.bin.installed.conda.has("caddy")
@@ -122,9 +115,13 @@ class Caddy {
     }
   }
   async uninstall(req, ondata) {
+    ondata({ raw: "cleaning up\r\n" })
     await this.kernel.bin.exec({
       message: `conda remove ${this.cmd()}`,
     }, ondata)
+    const folder = this.kernel.path("cache/XDG_DATA_HOME/caddy")
+    await rimraf(folder)
+    ondata({ raw: "finished cleaning up\r\n" })
   }
 }
 module.exports = Caddy

package/kernel/bin/conda.js

@@ -120,10 +120,9 @@ report_errors: false`)
 //  }
   async check() {
     let res = await this.kernel.bin.exec({ message: `conda list` }, (stream) => {
-      process.stdout.write(stream.raw)
+//      process.stdout.write(stream.raw)
 //        console.log("conda list check", { stream })
     })
-    console.log("2 conda list check", res)
 
     let lines = res.response.split(/[\r\n]+/)
     let conda_check = {}
@@ -179,7 +178,6 @@ report_errors: false`)
         }
       }
     }
-    console.log("> Check", { conda_check, conda })
     this.kernel.bin.installed.conda = conda
     return conda_check.conda && conda_check.mamba && conda_check.sqlite
     //return conda_check.conda && conda_check.mamba
@@ -254,16 +252,16 @@ report_errors: false`)
     // 3. so after conda update --all, it should be conda-libmamba-solver=24.11.1
 
     let mods = this.kernel.bin.mods.filter((m) => {
-      return ["zip", "uv", "node", "huggingface", "gxx", "git", "ffmpeg", "caddy"].includes(m.name)
+      return req.dependencies.includes(m.name)
+//      return ["zip", "uv", "node", "huggingface", "gxx", "git", "ffmpeg", "caddy"].includes(m.name)
     }).map((m) => {
       return m.mod.cmd()
     }).join(" ")
-    console.log("To install", { mods })
+    console.log("Conda dependencies to install", { mods })
 
     let cmds = [
       //"conda clean -y --index-cache",
       "conda clean -y --all",
-      //"conda install -y -c conda-forge sqlite=3.47.2",
       `conda install -y -c conda-forge sqlite=3.47.2 ${mods}`,
 
 //      `conda config --file ${this.kernel.path('condarc')} --set remote_connect_timeout_secs 20`,
@@ -377,9 +375,6 @@ report_errors: false`)
     let e
     for(let p of this.paths[this.kernel.platform]) {
       let e = await this.kernel.bin.exists(p)
-      console.log("kernel.bin.conda.installed")
-      console.log("e", e)
-      console.log("correct_conda", this.kernel.bin.correct_conda)
       if (e && this.kernel.bin.correct_conda) return true
     }
     return false

package/kernel/bin/index.js

@@ -28,6 +28,7 @@ const semver = require('semver')
 const Environment = require('../environment')
 //const imageToAscii = require("image-to-ascii");
 
+const Setup = require('./setup')
 
 
 //const Puppet = require("./puppeteer")
@@ -344,11 +345,8 @@ class Bin {
       this.correct_conda = false
     } else {
       res = await this.exec({ message: `conda list` }, (stream) => {
-        process.stdout.write(stream.raw)
   //        console.log("conda list check", { stream })
       })
-      console.log("1 conda list check", res)
-
 
       let lines = res.response.split(/[\r\n]+/)
       for(let line of lines) {
@@ -401,7 +399,6 @@ class Bin {
           }
         }
       }
-      console.log("conda_check", conda_check)
 
       if (conda_check.conda && conda_check.mamba && conda_check.sqlite) {
       //if (conda_check.conda && conda_check.mamba) {
@@ -440,33 +437,6 @@ class Bin {
       }
     }
 
-//    // 2. pip
-//    let pip = new Set()
-//    if (conda_exists) {
-//      // conda comes with pip
-//      console.log("## check pip")
-//      start = false
-//      res = await this.exec({ message: `pip list` }, (stream) => {
-////        console.log("pip list check", { stream })
-//      })
-//      console.log("PIP", res.response)
-//      lines = res.response.split(/[\r\n]+/)
-//      for(let line of lines) {
-//        if (start) {
-//          let chunks = line.split(/\s+/).filter(x => x)
-//          if (chunks.length > 1) {
-//            pip.add(chunks[0])
-//          }
-//        } else {
-//          if (/-------.*/i.test(line)) {
-//            start = true 
-//          }
-//        }
-//      }
-//    }
-//    this.installed.pip = pip
-    
-
     if (this.platform === "darwin") {
       // 3. brew
       console.log("## check brew")
@@ -668,11 +638,11 @@ class Bin {
     }
   }
   async install2(req, ondata) {
-    let { requirements, install_required, requirements_pending, error } = await this.check_bin()
+    let { requirements, install_required, requirements_pending, error } = await this.check({
+      bin: this.preset("ai")
+    })
     req.params = JSON.stringify(requirements)
-    console.log(">>>>> 1")
     if (this.install_required) {
-      console.log(">>>>> 2")
       let res = await this.install(req, ondata)
       return res
     }
@@ -695,13 +665,7 @@ class Bin {
     } else {
       this.client = null
     }
-
-    console.log("install", req)
-
     let requirements = JSON.parse(req.params)
-
-    console.time("### Total Install")
-
     for(let x=0; x<10; x++) {
       console.log(`## Install Attempt ${x}`)
       let i = 0;
@@ -715,9 +679,7 @@ class Bin {
           console.log("already installed. skip.")
         } else {
           console.log("not installed. install.")
-          console.time("############# Install " +requirement.name)
           await this.tryInstall(req.id, requirement, x, i, requirements.length, ondata)
-          console.timeEnd("############# Install " +requirement.name)
         }
       }
 
@@ -773,28 +735,38 @@ class Bin {
     //    }
     //  }
     //}
-    console.timeEnd("### Total Install")
     if (this.kernel.shell) {
       console.log(">>> shell reset")
       this.kernel.shell.reset()
     }
     await this.init()
   }
-  async installed(r) {
+  async check_installed(r, dependencies) {
     if (Array.isArray(r.name)) {
       for(let name of r.name) {
-        let installed = await this._installed(name, r.type)
+        let installed = await this._installed(name, r.type, dependencies)
         if (!installed) return false
       }
       return true
     } else {
-      let installed = await this._installed(r.name, r.type)
+      let installed = await this._installed(r.name, r.type, dependencies)
       return installed
     }
   }
-  async _installed(name, type) {
+  async _installed(name, type, dependencies) {
     if (type === "conda") {
-      return this.installed.conda.has(name)
+      let conda_installed = this.installed.conda.has(name)
+      console.log({ dependencies })
+      let dependencies_installed = true
+      for(let d of dependencies) {
+        console.log("checking dependency", d)
+        if(!this.installed.conda.has(d)) {
+          dependencies_installed = false
+          console.log("not installed", d)
+          break
+        }
+      }
+      return conda_installed && dependencies_installed
     } else if (type === "pip") {
       return this.installed.pip && this.installed.pip.has(name)
     } else if (type === "brew") {
@@ -810,50 +782,205 @@ class Bin {
       return installed
     }
   }
-  async check_bin() {
-    let requirements = [
-      { name: "conda", },
-      { name: "git", },
-      { name: "zip", },
-      { name: "node", },
-      { name: "ffmpeg", },
-      { name: "caddy", }
-    ]
+  preset(mode) {
+    return Setup[mode](this.kernel)
+  }
+  requirements(config) {
+    let requirements = config.bin.requirements
+    if (config.script && config.script.requires && config.script.requires.length > 0) {
+      /*********************************************************************
+
+      {
+        requires: [{
+          platform,
+          type,
+          name,
+          args
+        }, {
+          platform,
+          type,
+          name,
+          args
+        }],
+        run: [{
+          ...
+        }]
+      }
+
+      syntax :=
+
+        {
+          platform: <win32|darwin|linux>,
+          type: <conda|pip|brew|none>,
+          name: <package name>,           (example: "ffmpeg", "git")
+          args: <install command flags>   (example: "-c conda-forge")
+        }
+
+
+      1. pinokio native install: no need for specifying platforms since they are included
+
+        {
+          name: "conda"
+        }
+
+
+      2. non native install (conda, pip, brew)
+
+        2.1. Same on all platforms 
+
+        [{
+          type: "conda",
+          name: "ffmpeg",
+          args: "-c conda-forge"
+        }]
+
+        2.2. Specify per platform
+
+
+        [
+          { name: "conda" },
+          { platform: "darwin", type: "brew", name: "llvm" },
+          { platform: "linux", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" },
+          { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
+        ]
+
+        [
+          { name: "conda" },
+          { platform: ["darwin", "linux"], type: "brew", name: "llvm" },
+          { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
+        ]
+
+
+
+      *********************************************************************/
+      let type_name_set = new Set()
+      for(let r of config.script.requires) {
+        // if no platform specified, or if the specified platform matches the current platform
+        if (!r.platform || platform === r.platform || Array.isArray(r.platform) && r.platform.includes(platform) ) {
+          if (Array.isArray(r.name)) {
+            // if array, just add it
+            requirements.push(r)
+          } else {
+            let type_name = `${r.type ? r.type : ''}/${r.name}`
+            if (!type_name_set.has(type_name)) {
+              type_name_set.add(type_name)
+              requirements.push(r)
+            }
+          }
+        }
+      }
+
+    }
+    return requirements
+  }
+  relevant(r) {
+    /*
+      single platform
+      [
+        { name: "conda" },
+        { platform: "darwin", type: "brew", name: "llvm" },
+        { platform: "linux", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" },
+        { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
+      ]
+
+      multiple platforms
+      [
+        { name: "conda" },
+        { platform: ["darwin", "linux"], type: "brew", name: "llvm" },
+        { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
+      ]
+
+
+      platform & arch
+      [
+        { name: "conda" },
+        { platform: "darwin", arch: ["arm64", "x64"], type: "brew", name: "llvm" },
+      ]
+    */
+
     let platform = os.platform()
-    if (platform === "win32") {
-      requirements.push({ name: "registry" })
-      requirements.push({ name: "vs" })
+    let arch = os.arch()
+    let gpu = this.kernel.gpu
+    let relevant = {
+      platform: false,
+      arch: false,
+      gpu: false,
     }
-    if (platform === "darwin") {
-      requirements.push({ name: "brew" })
+    if (r.platform) {
+      if (Array.isArray(r.platform)) {
+        // multiple items
+        if (r.platform.includes(platform)) {
+          relevant.platform = true
+        }
+      } else {
+        // one item
+        if (r.platform === platform) {
+          relevant.platform = true
+        }
+      }
+    } else {
+      // all platforms
+      relevant.platform = true
     }
-    if (platform === "linux") {
-      requirements.push({ name: "gxx" })
+    if (r.arch) {
+      if (Array.isArray(r.arch)) {
+        // multiple items
+        if (r.arch.includes(arch)) {
+          relevant.arch = true
+        }
+      } else {
+        // one item
+        if (r.arch === arch) {
+          relevant.arch = true
+        }
+      }
+    } else {
+      // all platforms
+      relevant.arch = true
     }
-    if (this.kernel.gpu === "nvidia") {
-      requirements.push({ name: "cuda", })
+    if (r.gpu) {
+      if (Array.isArray(r.gpu)) {
+        // multiple items
+        if (r.gpu.includes(gpu)) {
+          relevant.gpu = true
+        }
+      } else {
+        // one item
+        if (r.gpu === gpu) {
+          relevant.gpu = true
+        }
+      }
+    } else {
+      // all platforms
+      relevant.gpu = true
     }
-    requirements = requirements.concat([
-      { name: "cloudflared" },
-//      { name: "playwright" },
-      { name: "huggingface" },
-      { name: "uv" },
-      { name: "py" },
-    ])
-
+    return relevant.platform && relevant.arch && relevant.gpu
+  }
+  async check(config) {
+    let requirements = this.requirements(config)
     let requirements_pending = !this.installed_initialized
     let install_required = true
     if (!requirements_pending) {
       install_required = false
       for(let i=0; i<requirements.length; i++) {
         let r = requirements[i]
-        console.time(r.name)
-        let installed = await this.installed(r)
-        console.log({ r, installed })
-        console.timeEnd(r.name)
-        requirements[i].installed = installed
-        if (!installed) {
-          install_required = true
+        //let installed = await this.installed(r)
+        //requirements[i].installed = installed
+        //if (!installed) {
+        //  install_required = true
+        //}
+        let relevant = this.relevant(r)
+        requirements[i].relevant = relevant
+        if (relevant) {
+          let dependencies
+          if (r.name === "conda") {
+            dependencies = config.bin.conda_requirements
+          }
+          let installed = await this.check_installed(r, dependencies)
+          requirements[i].installed = installed
+          if (!installed) {
+            install_required = true
+          }
         }
       }
     }
@@ -870,6 +997,11 @@ class Bin {
     this.requirements_pending = requirements_pending
 
     console.log("check_bin finished 2")
+
+    requirements = requirements.filter((r) => {
+      return r.relevant
+    })
+
     return {
       error,
       requirements,

package/kernel/bin/setup.js

@@ -0,0 +1,130 @@
+const os = require('os')
+let platform = os.platform()
+module.exports = {
+  ai: (kernel) => {
+    let conda_requirements = [
+      "uv",
+      "node",
+      "huggingface",
+      "git",
+      "ffmpeg",
+//      "caddy"
+    ]
+    let requirements = [
+      { name: "conda", },
+      { name: "git", },
+  //      { name: "zip", },
+      { name: "node", },
+      { name: "ffmpeg", },
+//      { name: "caddy", }
+    ]
+    if (platform === "win32") {
+      requirements.push({ name: "registry" })
+      requirements.push({ name: "vs" })
+    }
+    if (platform === "darwin") {
+      requirements.push({ name: "brew" })
+    }
+    if (platform === "linux") {
+      requirements.push({ name: "gxx" })
+      conda_requirements.push("gxx")
+    }
+    if (kernel.gpu === "nvidia") {
+      requirements.push({ name: "cuda", })
+      conda_requirements.push("cuda")
+    }
+    requirements = requirements.concat([
+      { name: "cloudflared" },
+//      { name: "playwright" },
+      { name: "huggingface" },
+      { name: "uv" },
+      { name: "py" },
+    ])
+    return {
+      icon: "fa-solid fa-brain",
+      title: "AI",
+      description: "Install common modules required for running AI locally",
+      requirements,
+      conda_requirements
+    }
+  },
+  javascript: (kernel) => {
+    return {
+      icon: "fa-brands fa-js",
+      title: "Node.js",
+      description: "Set up self-contained node.js development environment",
+      requirements: [
+        { name: "conda", },
+        { name: "git", },
+        { name: "node", },
+      ],
+      conda_requirements: [
+        "node",
+        "git",
+      ]
+    }
+  },
+  python: (kernel) => {
+    return {
+      icon: "fa-brands fa-python",
+      title: "Python",
+      description: "Set up self-contained python development environment",
+      requirements: [
+        { name: "conda", },
+        { name: "git", },
+        { name: "uv", },
+      ],
+      conda_requirements: [
+        "uv",
+        "git",
+      ]
+    }
+  },
+  dev: (kernel) => {
+    let requirements = [
+      { name: "conda", },
+      { name: "git", },
+      { name: "node", },
+      { name: "uv", },
+    ]
+    let conda_requirements = [
+      "uv",
+      "node",
+      "git",
+    ]
+    if (platform === "win32") {
+      requirements.push({ name: "registry" })
+      requirements.push({ name: "vs" })
+    }
+    if (platform === "darwin") {
+      requirements.push({ name: "brew" })
+    }
+    if (platform === "linux") {
+      requirements.push({ name: "gxx" })
+      conda_requirements.push("gxx")
+    }
+    return {
+      icon: "fa-solid fa-laptop-code",
+      title: "Coding",
+      description: "Install common modules required for development (Node.js, python, Visual Studio Developer Tools (Windows), Xcode build tools (Mac)",
+      requirements,
+      conda_requirements,
+    }
+  },
+  network: (kernel) => {
+    return {
+      icon: "fa-solid fa-wifi",
+      title: "Network",
+      description: "Automatic HTTPS and local network sharing for ALL local apps",
+      requirements: [
+        { name: "conda", },
+        { name: "git", },
+        { name: "caddy", },
+      ],
+      conda_requirements: [
+        "git",
+        "caddy"
+      ]
+    }
+  }
+}