pinokiod 3.8.147 → 3.8.149

package/server/index.js

@@ -47,6 +47,8 @@ const Environment = require("../kernel/environment")
 const Cloudflare = require("../kernel/api/cloudflare")
 const Util = require("../kernel/util")
 
+const Setup = require("../kernel/bin/setup")
+
 class Server {
   constructor(config) {
     this.tabs = {}
@@ -428,6 +430,21 @@ class Server {
       rawpath,
     })
   }
+  getVariationUrls(req) {
+    let edu = new URL("http://localhost" + req.originalUrl)
+    edu.searchParams.set("mode", "source")
+    let editorUrl = edu.pathname + edu.search
+
+    let referer = req.get("Referer")
+    let prevUrl = null
+    try {
+      if (/\/env\/api\/.+/.test(new URL(referer).pathname)) {
+        prevUrl = referer 
+      }
+    } catch (e) {
+    }
+    return { editorUrl, prevUrl }
+  }
   async render(req, res, pathComponents, meta) {
 
     
@@ -556,8 +573,9 @@ class Server {
         display: ["form"]
       })
     } else if (pathComponents.length === 0 && req.query.mode === "download") {
-      let { requirements, install_required, requirements_pending, error } = await this.kernel.bin.check_bin()
-
+      let { requirements, install_required, requirements_pending, error } = await this.kernel.bin.check({
+        bin: this.kernel.bin.preset("ai"),
+      })
       res.render("download", {
         error,
         current: req.originalUrl,
@@ -762,240 +780,57 @@ class Server {
           resolved = runner
         }
 
-        let template
-        template = "terminal"
-
+        let template = "terminal"
         if (req.query && req.query.mode === "source") {
           template = "editor"
         }
 
-
-// Deprecate source view and form view => Everything is full screen by default
-//        if (req.query && req.query.mode === "source") {
-//          if (req.query && req.query.fullscreen) {
-//            template = "fullscreen_editor"
-//          } else {
-//            template = "editor"
-//          }
-//        } else if (schemaPath && schemaPath.length > 0) {
-//          template = "form"
-//        } else {
-//          if (req.query && req.query.fullscreen) {
-//            template = "fullscreen_editor"
-//          } else {
-//            template = "editor"
-//          }
-//        }
-        
-
-        let requirements = [{
-          name: "conda",
-        }, {
-          name: "git",
-        }, {
-          name: "zip",
-        }, {
-          name: "node",
-        }, {
-          name: "ffmpeg",
-        }, {
-          name: "caddy",
-//          type: "conda",
-//          name: "ffmpeg",
-//          args: "-c conda-forge"
-        }]
-        let platform = os.platform()
-        if (platform === "win32") {
-          requirements.push({
-            name: "registry"
-          })
-          requirements.push({
-            name: "vs"
-          })
-        }
-        if (platform === "darwin") {
-          requirements.push({
-            name: "brew"
-          })
-        }
-        if (platform === "linux") {
-          requirements.push({
-            name: "gxx"
-          })
-        }
-        if (this.kernel.gpu === "nvidia") {
-          requirements.push({
-            name: "cuda"
-          })
-        }
-        requirements = requirements.concat([{
-          name: "cloudflared"
-//        }, {
-//          name: "playwright"
-        }, {
-          name: "huggingface"
-        }, {
-          name: "uv"
-        }, {
-          name: "py"
-        }])
-//        if (platform === "linux") {
-//          requirements.push({
-//            name: "brew"
-//          })
-//        }
-
-        if (resolved && resolved.requires && resolved.requires.length > 0) {
-          /*********************************************************************
-
-          syntax :=
-
-            {
-              platform: <win32|darwin|linux>,
-              type: <conda|pip|brew|none>,
-              name: <package name>,           (example: "ffmpeg", "git")
-              args: <install command flags>   (example: "-c conda-forge")
-            }
-
-
-          1. pinokio native install: no need for specifying platforms since they are included
-
-            {
-              name: "conda"
-            }
-
-
-          2. non native install (conda, pip, brew)
-
-            2.1. Same on all platforms 
-
-            [{
-              type: "conda",
-              name: "ffmpeg",
-              args: "-c conda-forge"
-            }]
-
-            2.2. Specify per platform
-
-
-            [
-              { name: "conda" },
-              { platform: "darwin", type: "brew", name: "llvm" },
-              { platform: "linux", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" },
-              { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
-            ]
-
-            [
-              { name: "conda" },
-              { platform: ["darwin", "linux"], type: "brew", name: "llvm" },
-              { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
-            ]
-
-
-
-          *********************************************************************/
-
-
-          let platform = os.platform()
-          let type_name_set = new Set()
-          for(let r of resolved.requires) {
-            // if no platform specified, or if the specified platform matches the current platform
-            if (!r.platform || platform === r.platform || Array.isArray(r.platform) && r.platform.includes(platform) ) {
-              if (Array.isArray(r.name)) {
-                // if array, just add it
-                requirements.push(r)
-              } else {
-                let type_name = `${r.type ? r.type : ''}/${r.name}`
-                if (!type_name_set.has(type_name)) {
-                  type_name_set.add(type_name)
-                  requirements.push(r)
-                }
-              }
-            }
-          }
-
-        }
-        let requirements_pending = !this.kernel.bin.installed_initialized
-
-        let install_required = true
-        if (!requirements_pending) {
-          install_required = false
-          for(let i=0; i<requirements.length; i++) {
-            let r = requirements[i]
-
-            let relevant = this.relevant(r)
-            requirements[i].relevant = relevant
-            if (relevant) {
-              let installed = await this.installed(r)
-              requirements[i].installed = installed
-              if (!installed) {
-                install_required = true
-              }
-            }
-          }
-        }
-
-        let error = null
-        try {
-          this.kernel.bin.compatible()
-        } catch (e) {
-          error = e.message
-          install_required = true
-        }
-
-        requirements = requirements.filter((r) => {
-          return r.relevant
+        let { requirements, install_required, requirements_pending, error } = await this.kernel.bin.check({
+          bin: this.kernel.bin.preset("ai"),
+          script: resolved
         })
 
+        //let requirements = this.kernel.bin.requirements(resolved)
+        //let requirements_pending = !this.kernel.bin.installed_initialized
+        //let install_required = true
+        //if (!requirements_pending) {
+        //  install_required = false
+        //  for(let i=0; i<requirements.length; i++) {
+        //    let r = requirements[i]
+
+        //    let relevant = this.relevant(r)
+        //    requirements[i].relevant = relevant
+        //    if (relevant) {
+        //      let installed = await this.installed(r)
+        //      requirements[i].installed = installed
+        //      if (!installed) {
+        //        install_required = true
+        //      }
+        //    }
+        //  }
+        //}
 
-        let mem = this.getMemory(filepath)
-        let edu = new URL("http://localhost" + req.originalUrl)
-        edu.searchParams.set("mode", "source")
-        let editorUrl = edu.pathname + edu.search
+        //let error = null
+        //try {
+        //  this.kernel.bin.compatible()
+        //} catch (e) {
+        //  error = e.message
+        //  install_required = true
+        //}
 
-        let referer = req.get("Referer")
+        //requirements = requirements.filter((r) => {
+        //  return r.relevant
+        //})
 
-        let prev = null
-        try {
-          if (/\/env\/api\/.+/.test(new URL(referer).pathname)) {
-            prev = referer 
-          }
-        } catch (e) {
-        }
 
+        let mem = this.getMemory(filepath)
 
-        let requires_instantiation = false
-        let pre_items = []
-        if (resolved && resolved.pre) {
-          let env = await Environment.get2(filepath, this.kernel)
-          for(let item of resolved.pre) {
-            if (item.env) {
+        let { editorUrl, prevUrl } = this.getVariationUrls(req)
 
-              if (!item.index) {
-                item.index = 0
-              }
-              if (item.key) {
-                const hasProtocol = /^[a-zA-Z][a-zA-Z\d+\-.]*:\/\//.test(item.key);
-                const url = new URL(hasProtocol ? item.key : `https://${item.key}`);
-                item.host = url.host
-                item.val = await this.kernel.kv.get(item.host, item.index)
-              } else {
-                item.host = ""
-              }
 
-              if (env[item.env]) {
-                item.val = env[item.env]
-              } else {
-                if (item.default) {
-                  item.val = item.default
-                }
-                requires_instantiation = true
-              }
-              pre_items.push(item)
-            }
-          }
-        }
-        if (requires_instantiation) {
+        let env_requirements = await Environment.requirements(resolved, filepath, this.kernel)
+        console.log({ env_requirements })
+        if (env_requirements.requires_instantiation) {
           let p = Util.api_path(filepath, this.kernel)
           let platform = os.platform()
           if (platform === "win32") {
@@ -1006,12 +841,12 @@ class Server {
             theme: this.theme,
             filename,
             filepath: p,
-            items: pre_items
+            items: env_requirements.items
           })
         } else {
           let logpath = encodeURIComponent(Util.log_path(filepath, this.kernel))
           const result = {
-            prev,
+            prev: prevUrl,
             error,
             memory: mem,
   //          memory: mem,
@@ -1762,89 +1597,6 @@ class Server {
       return config
     }
   }
-  relevant(r) {
-    /*
-      single platform
-      [
-        { name: "conda" },
-        { platform: "darwin", type: "brew", name: "llvm" },
-        { platform: "linux", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" },
-        { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
-      ]
-
-      multiple platforms
-      [
-        { name: "conda" },
-        { platform: ["darwin", "linux"], type: "brew", name: "llvm" },
-        { platform: "win32", tyoe: "conda", name: "llvm", args: "-c conda-forge llvm" }
-      ]
-
-
-      platform & arch
-      [
-        { name: "conda" },
-        { platform: "darwin", arch: ["arm64", "x64"], type: "brew", name: "llvm" },
-      ]
-    */
-
-    let platform = os.platform()
-    let arch = os.arch()
-    let gpu = this.kernel.gpu
-    let relevant = {
-      platform: false,
-      arch: false,
-      gpu: false,
-    }
-    if (r.platform) {
-      if (Array.isArray(r.platform)) {
-        // multiple items
-        if (r.platform.includes(platform)) {
-          relevant.platform = true
-        }
-      } else {
-        // one item
-        if (r.platform === platform) {
-          relevant.platform = true
-        }
-      }
-    } else {
-      // all platforms
-      relevant.platform = true
-    }
-    if (r.arch) {
-      if (Array.isArray(r.arch)) {
-        // multiple items
-        if (r.arch.includes(arch)) {
-          relevant.arch = true
-        }
-      } else {
-        // one item
-        if (r.arch === arch) {
-          relevant.arch = true
-        }
-      }
-    } else {
-      // all platforms
-      relevant.arch = true
-    }
-    if (r.gpu) {
-      if (Array.isArray(r.gpu)) {
-        // multiple items
-        if (r.gpu.includes(gpu)) {
-          relevant.gpu = true
-        }
-      } else {
-        // one item
-        if (r.gpu === gpu) {
-          relevant.gpu = true
-        }
-      }
-    } else {
-      // all platforms
-      relevant.gpu = true
-    }
-    return relevant.platform && relevant.arch && relevant.gpu
-  }
   async _installed(name, type) {
     if (type === "conda") {
       return this.kernel.bin.installed.conda.has(name)
@@ -1933,6 +1685,85 @@ class Server {
     }
     console.log("FINISHED MV")
   }
+  getPeerInfo() {
+    let list = []
+    let peers_info = {}
+    if (this.kernel.peer.info) {
+      peers_info = this.kernel.peer.info
+      let remote_peers = Object.keys(this.kernel.peer.info).filter(x => x !== this.kernel.peer.host)
+      let nodes = [this.kernel.peer.host].concat(remote_peers)
+      for(let host of nodes) {
+        let processes = []
+        try {
+          let procs = this.kernel.peer.info[host].proc
+          let router = this.kernel.peer.info[host].router
+          let port_mapping = this.kernel.peer.info[host].port_mapping
+          for(let proc of procs) {
+            let chunks = proc.ip.split(":")
+            let internal_port = chunks[chunks.length-1]
+            let internal_host = chunks.slice(0, chunks.length-1).join(":")
+            let external_port = port_mapping[internal_port]
+
+            let merged
+            let external_ip
+            if (external_port) {
+              external_ip = `${host}:${external_port}`
+//              merged = Array.from(new Set(router[external_ip].concat(router[proc.ip])))
+            } else {
+//              merged = router[proc.ip]
+            }
+            processes.push({
+              external_router: router[external_ip] || [],
+              internal_router: router[proc.ip] || [],
+              //router: merged || [],
+              external_ip,
+              external_port: parseInt(external_port),
+              internal_port: parseInt(internal_port),
+              ...proc
+            })
+            //if (external_port) {
+            //  let external_ip = `${host}:${external_port}`
+            //  // merge router
+            //  let merged = Array.from(new Set(router[external_ip].concat(router[proc.ip))
+            //  processes.push({
+            //    //proxy: this.kernel.caddy.mapping[item.ip] || [],
+            //    //router: router[external_ip] || [],
+            //    router: merged || [],
+            //    external_ip,
+            //    external_port: parseInt(external_port),
+            //    internal_port: parseInt(internal_port),
+            //    ...proc
+            //  })
+            //} else {
+            //  processes.push({
+            //    router: [],
+            //    external_port: parseInt(external_port),
+            //    internal_port: parseInt(internal_port),
+            //    ...proc
+            //  })
+            //}
+          }
+          // merge processes
+          // 1. 
+          processes.sort((a, b) => {
+            return b.external_port-a.external_port
+          })
+          list.push({
+            host,
+            name: this.kernel.peer.info[host].name,
+            platform: this.kernel.peer.info[host].platform,
+            processes
+          })
+        } catch (e) {
+          console.log(">>e", e)
+          console.log({ host, info: this.kernel.peer.info })
+        }
+      }
+//      console.log("Loaded yet?", nodes.length, Object.keys(peers_info).length, nodes.length === Object.keys(peers_info).length)
+    }
+    return list
+  }
+
 
   async syncConfig() {
 
@@ -2391,6 +2222,15 @@ class Server {
     //let home = this.kernel.homedir
     //let home = this.kernel.store.get("home")
     this.app.get("/", ex(async (req, res) => {
+      // check bin folder
+      let bin_path = this.kernel.path("bin/miniconda")
+      let bin_exists = await this.exists(bin_path)
+      if (!bin_exists) {
+        res.redirect("/setup")
+        return
+      }
+      
+
       if (req.query.mode !== "settings" && !home) {
         res.redirect("/?mode=settings")
         return
@@ -2625,67 +2465,85 @@ class Server {
         id: req.params.id
       })
     }))
-    this.app.get("/network", ex(async (req, res) => {
-      let list = []
-
-//      let peers_info = await this.kernel.peer.refresh()
-      let peers_info = {}
-      if (this.kernel.peer.info) {
-        peers_info = this.kernel.peer.info
-//        let info = this.kernel.router.info()
-//        console.log("33 info", info)
-        let remote_peers = Object.keys(this.kernel.peer.info).filter(x => x !== this.kernel.peer.host)
-        let nodes = [this.kernel.peer.host].concat(remote_peers)
-        for(let host of nodes) {
-          let processes = []
-          try {
-            let procs = this.kernel.peer.info[host].proc
-            let router = this.kernel.peer.info[host].router
-            console.log({ router })
-            let port_mapping = this.kernel.peer.info[host].port_mapping
-            for(let proc of procs) {
-              let internal_port = proc.ip.split(":")[1]
-              let external_port = port_mapping[internal_port]
-              if (external_port) {
-                let external_ip = `${host}:${external_port}`
-                console.log({ internal_port, external_port, external_ip })
-                processes.push({
-                  //proxy: this.kernel.caddy.mapping[item.ip] || [],
-                  router: router[external_ip] || [],
-                  external_ip,
-                  external_port: parseInt(external_port),
-                  internal_port: parseInt(internal_port),
-                  ...proc
-                })
-              } else {
-                processes.push({
-                  router: [],
-                  external_port: parseInt(external_port),
-                  internal_port: parseInt(internal_port),
-                  ...proc
-                })
-              }
-            }
-            processes.sort((a, b) => {
-              return b.external_port-a.external_port
-            })
-            list.push({
-              host,
-              name: this.kernel.peer.info[host].name,
-              platform: this.kernel.peer.info[host].platform,
-              processes
-            })
-          } catch (e) {
-            console.log(">>e", e)
-            console.log({ host, info: this.kernel.peer.info })
-          }
+    this.app.get("/peer_check", ex(async (req, res) => {
+      if (this.kernel.peer.refreshing) {
+        res.json({ updated: false })
+      } else {
+        let list = this.getPeerInfo()
+        if (JSON.stringify(this.last_list) !== JSON.stringify(list)) {
+          this.last_list = list
+          res.json({ updated: true })
+        } else {
+          res.json({ updated: false })
         }
       }
-      console.log({ peers_info })
+    }))
+    this.app.get("/setup", ex(async (req, res) => {
+      let items = []
+      for(let id in Setup) {
+        let item = Setup[id](this.kernel)
+        items.push({
+          id,
+          ...item
+        })
+      }
+      res.render("setup_home", {
+        items,
+        logo: this.logo,
+        theme: this.theme,
+        agent: this.agent,
+      })
+    }))
+    this.app.get("/setup/:mode", ex(async (req, res) => {
+      /*
+      1. mode:ai => all
+      2. mode:coding => conda, nodejs, git
+      3. mode:network => conda, git, caddy
+      4. mode: 
+      */
+
+      let bin = this.kernel.bin.preset(req.params.mode)
+
+      let { requirements, install_required, requirements_pending, error } = await this.kernel.bin.check({
+        bin
+      })
+      // set dependencies for conda
+      for(let i=0; i<requirements.length; i++) {
+        let r = requirements[i]
+        if (r.name === "conda") {
+          requirements[i].dependencies = bin.conda_requirements
+        }
+      }
+      console.log("RE", JSON.stringify(requirements, null, 2))
+      let current = req.query.callback || req.originalUrl
+      res.render("setup", {
+        error,
+        current,
+        install_required,
+        requirements,
+        requirements_pending,
+        logo: this.logo,
+        theme: this.theme,
+        agent: this.agent,
+      })
+    }))
+    this.app.get("/network", ex(async (req, res) => {
+      let { requirements, install_required, requirements_pending, error } = await this.kernel.bin.check({
+        bin: this.kernel.bin.preset("network"),
+      })
+      console.log({ requirements, install_required, requirements_pending })
+
+      if (install_required) {
+        res.redirect("/setup/network?callback=/network")
+        return
+      }
+
+      let list = this.getPeerInfo()
+
 
       let peers = []
-      for(let host in peers_info) {
-        let peer_info = peers_info[host]
+      for(let host in this.kernel.peer.info) {
+        let peer_info = this.kernel.peer.info[host]
         peers.push({
           host,
           name: peer_info.name,
@@ -3308,7 +3166,6 @@ class Server {
       res.json({ success: true })
     }))
     this.app.get("/pinokio/peer", ex(async (req, res) => {
-      console.log("/pinokio/peer", { proc: this.kernel.processes.info })
 //      await this.kernel.refresh()
       res.json({
         home: this.kernel.homedir,
@@ -3566,11 +3423,11 @@ class Server {
       }
       res.redirect(webpath)
     }))
-    this.app.get("/pinokio/folder/:id", ex(async (req, res) => {
-      let folder = this.kernel.proto.kv[req.params.id]
-      res.json(folder)
+    this.app.get("/pinokio/prototype/*", ex(async (req, res) => {
+      let id = req.params[0]
+      res.json(this.kernel.proto.kv[id])
     }))
-    this.app.post("/pinokio/create_folder", this.upload.any(), ex(async (req, res) => {
+    this.app.post("/pinokio/prototype", this.upload.any(), ex(async (req, res) => {
       try {
         /*
           {
@@ -3588,7 +3445,7 @@ class Server {
         console.log({ formData })
         // 1. instantiate based on id
         let script = this.kernel.proto.kv[formData.id]
-        await script.run(this.kernel, formData)
+        await script.create(this.kernel, formData)
         // 2. create pinokio_meta.json
         await this.createMeta(formData)
         res.json({ success: true })
@@ -3871,7 +3728,6 @@ class Server {
 
     }))
     this.app.get("/pinokio/requirements_ready", ex((req, res) => {
-
       let requirements_pending = !this.kernel.bin.installed_initialized
       console.log({ requirements_pending })
       res.json({ requirements_pending })
@@ -3946,7 +3802,6 @@ class Server {
       })
     });
     process.on('SIGINT', () => {
-      console.log("[SigInt event] Kill", process.pid)
       //if (this.kernel && this.kernel.shell) {
       //  console.log("shell reset")
       //  this.kernel.shell.reset(() => {
@@ -3955,7 +3810,12 @@ class Server {
       //} else {
       //  process.exit()
       //}
-      let map = this.kernel.processes.map || {}
+      console.log("[SigInt event] Kill", process.pid)
+      if (this.kernel.processes.caddy_pid) {
+        console.log("kill caddy", this.kernel.processes.caddy_pid)
+        kill(this.kernel.processes.caddy_pid, "SIGKILL", true)
+      }
+      console.log("kill self")
       kill(process.pid, 'SIGKILL', true)
       //kill(process.pid, map, 'SIGKILL', () => {
       //  console.log("child procs killed for", process.pid)
@@ -3964,7 +3824,6 @@ class Server {
     })
 
     process.on('SIGTERM', () => {
-      console.log("[Sigterm event] Kill", process.pid)
 //      if (this.kernel && this.kernel.shell) {
 //        console.log("shell reset")
 //        this.kernel.shell.reset(() => {
@@ -3973,6 +3832,12 @@ class Server {
 //      } else {
 //        process.exit()
 //      }
+      console.log("[Sigterm event] Kill", process.pid)
+      if (this.kernel.processes.caddy_pid) {
+        console.log("kill caddy", this.kernel.processes.caddy_pid)
+        kill(this.kernel.processes.caddy_pid, "SIGKILL", true)
+      }
+      console.log("kill self")
       kill(process.pid, 'SIGKILL', true)
       //let map = this.kernel.processes.map || {}
       //kill(process.pid, map, 'SIGKILL', () => {