pinokiod 1.3.28 → 1.3.30

package/kernel/api/fs/index.js

@@ -8,8 +8,21 @@ const { DownloaderHelper } = require('node-downloader-helper');
 const randomUseragent = require('random-useragent');
 const symlinkDir = require('symlink-dir')
 const retry = require('async-retry');
+const { createHash } = require('crypto');
 
 
+const log = (msgs, ondata) => {
+  for(let msg of msgs) {
+    if (typeof msg === "object") {
+      msg = JSON.stringify(msg, null, 2).replace(/\n/g, "\r\n")
+    }
+    if (ondata) {
+      ondata({ raw: "\r\n" + msg })
+    }
+    console.log(msg)
+  }
+}
+
 class FS {
   async read(req, ondata, kernel) {
     /*
@@ -133,6 +146,7 @@ class FS {
         [p],
         ondata
       )
+      log([res], ondata)
       // symlink:
       //  from => :package_path
       //  to => /drives/packages/pip/:package_name/:package_version
@@ -174,19 +188,76 @@ class FS {
         const copy = res[name].copy
         const move = res[name].move
         const version = res[name].version
+        const url = res[name].url
 
         if (version) {
+
+          let namespace
+          // handle custom install from url
+          log(["> Namespace resolution"], ondata)
+          if (url) {
+            log(["> Custom url exists. Resolve custom namespace..."], ondata)
+            if (res[name].archive_info) {
+              let info = res[name].archive_info
+              log(["> Archive: ", { url, info }], ondata)
+              // try to get sha256 first
+              if (info.hashes) {
+                if (info.hashes.sha256) {
+                  let hash = info.hashes.sha256
+                  namespace = `${name}/${version}-f-${hash.slice(0, 4)}`
+                } else {
+                  let keys = Object.keys(info.hashes)
+                  // just use the first hash
+                  let key = keys[0]
+                  let hash = info.hashes[key]
+                  namespace = `${name}/${version}-${hash.slice(0, 4)}`
+                }
+              }
+              // otherwise get any other hash
+            } else if (res[name].vcs_info) {
+              let info = res[name].vcs_info
+              log(["> VCS: ", { url, info }], ondata)
+              namespace = `${name}/${version}-${info.vcs}-${info.commit_id.slice(0, 4)}`
+            } else if (res[name].dir_info) {
+              let info = res[name].dir_info
+              log(["> Directory: ", { url, info }], ondata)
+              let hash = createHash('sha256').update(url).digest('hex')
+              namespace = `${name}/${version}-d-${hash.slice(0, 4)}`
+            }
+
+            // IMPORTANT: if the namespace doesn't get resolved, do NOT create a drive 
+            if (!namespace) {
+              continue
+            }
+
+          } else {
+            namespace = `${name}/${version}`
+          }
+
+          log(["> Resolved namespace", namespace], ondata)
+
           let site_packages_root;
           if (kernel.platform === "win32") {
-            site_packages_root = `pip/${name}/${version}/Lib/site-packages`
+            site_packages_root = `pip/${namespace}/Lib/site-packages`
           } else {
-            site_packages_root = `pip/${name}/${version}/lib/python3.10/site-packages`
+            site_packages_root = `pip/${namespace}/lib/python3.10/site-packages`
           }
           const d = {
             copy: {},
             move: {},
-            parent: site_packages_root
+            parent: site_packages_root,
           }
+
+          // if the version is NOT final, overwrite.
+          let re = /^\d+(\.\d+)*$/        // regex for testing final versions like 1.2, 1.1.1, 1.2.1.1, etc.
+          if (re.test(version)) {
+            // final version
+          } else {
+            d.options = {
+              overwrite: true
+            }
+          }
+
           for(let relpath of copy) {
             d.copy[relpath] = kernel.path("drive", p, relpath)
           }

package/kernel/bin/conda.js

@@ -51,6 +51,31 @@ class Conda {
     }
     return base
   }
+  async init() {
+    // 
+    if (this.kernel.homedir) {
+      let exists = await this.kernel.exists("condarc")
+      if (!exists) {
+        await fs.promises.writeFile(this.kernel.path('condarc'), `channels:
+  - conda-forge
+  - defaults
+create_default_packages:
+  - python=3.10`)
+      }
+    }
+  }
+//  async init() {
+//    let exists = await this.kernel.bin.exists("miniconda/condarc")
+//    console.log("condarc exists?", exists)
+//    if (!exists) {
+//      console.log("write to condarc")
+//      await fs.promises.writeFile(this.kernel.bin.path('miniconda/condarc'), `channels:
+//    - conda-forge
+//    - defaults
+//  create_default_packages:
+//    - python=3.10`)
+//    }
+//  }
   async install(req, ondata) {
     const installer_url = this.urls[this.kernel.platform][this.kernel.arch]
     const installer = this.installer[this.kernel.platform]
@@ -73,12 +98,12 @@ class Conda {
     await this.kernel.bin.exec({ message: cmd, conda: { skip: true } }, (stream) => {
       ondata(stream)
     })
-//    await this.activate()
-    await fs.promises.writeFile(this.kernel.bin.path('miniconda/condarc'), `channels:
-  - conda-forge
-  - defaults
-create_default_packages:
-  - python=3.10`)
+////    await this.activate()
+//    await fs.promises.writeFile(this.kernel.bin.path('miniconda/condarc'), `channels:
+//  - conda-forge
+//  - defaults
+//create_default_packages:
+//  - python=3.10`)
     await this.kernel.bin.exec({
       message: [
 //        (this.kernel.platform === 'win32' ? 'conda_hook' : `eval "$(conda shell.bash hook)"`),
@@ -91,8 +116,10 @@ create_default_packages:
         `conda config --file ${this.kernel.bin.path('miniconda', 'condarc')} --add create_default_packages python=3.10`,
         */
         //"conda update -y conda",
+        //"conda update -n base -c conda-forge -c defaults conda",
 
 
+        //"conda install conda=24.5.0",
         "conda update -y --all",
 
 

package/kernel/index.js

@@ -102,8 +102,12 @@ class Kernel {
     return path.resolve(this.homedir, ...args)
   }
   exists(...args) {
-    let abspath = this.path(...args)
-    return new Promise(r=>fs.access(abspath, fs.constants.F_OK, e => r(!e)))
+    if (args) {
+      let abspath = this.path(...args)
+      return new Promise(r=>fs.access(abspath, fs.constants.F_OK, e => r(!e)))
+    } else {
+      return false
+    }
   }
   async load(...filepath) {
     let p = path.resolve(...filepath)
@@ -171,6 +175,11 @@ class Kernel {
   async init() {
     let home = this.store.get("home")
 
+    // reset shells if they exist
+    if (this.shell) {
+      this.shell.reset()
+    }
+
     this.homedir = home
 
 //    if (home) {

package/kernel/shell.js

@@ -125,7 +125,7 @@ class Shell {
     this.env.XDG_STATE_HOME = path.resolve(this.kernel.homedir, "cache", "XDG_STATE_HOME")
     this.env.GRADIO_TEMP_DIR = path.resolve(this.kernel.homedir, "cache", "GRADIO_TEMP_DIR")
     this.env.PIP_CONFIG_FILE = path.resolve(this.kernel.homedir, "pipconfig")
-    this.env.CONDARC = path.resolve(this.kernel.homedir, "bin", "miniconda", "condarc")
+    this.env.CONDARC = path.resolve(this.kernel.homedir, "condarc")
 
     this.env.PS1 = "<<PINOKIO SHELL>> "
 //    this.env.PROMPT_COMMAND = "export PS1=\"<<PINOKIO SHELL>> \""
@@ -269,7 +269,6 @@ class Shell {
       })
       return response
     } else {
-      console.log("SHELL", { params, env: this.env })
       let response = await this.request(params, async (stream) => {
         if (stream.prompt) {
           this.resolve()
@@ -741,14 +740,14 @@ class Shell {
 
     */
     let info = {
-      id: this.id,
+      path: this.path,
+      cmd: this.cmd,
       index: this.index,
       group: this.group,
       env: this.env,
-      path: this.path,
-      cmd: this.cmd,
       done: this.done,
       ready: this.ready,
+      id: this.id,
       ts: Date.now()
     }
 
@@ -784,12 +783,8 @@ ${buf}
 
     data.cleaned = `######################################################################
 #
-# group: ${this.group}
-# id: ${this.id}
-# index: ${this.index}
-# cmd: ${this.cmd}
 # timestamp: ${time}
-#
+# ${infoYAML.replaceAll("\n", "\n# ")}
 
 ${cleaned}
 

package/kernel/shells.js

@@ -197,6 +197,13 @@ class Shells {
   stop (request, message) {
     return this.kill(request, message)
   }
+  reset() {
+    if (this.shells) {
+      for(let shell of this.shells) {
+        shell.kill("", true)
+      }
+    }
+  }
   kill(request, message) {
     /*
     *  - Kill by ID

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "1.3.28",
+  "version": "1.3.30",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -47,7 +47,7 @@
     "node-gradio-client": "^0.14.6",
     "node-pty-prebuilt-multiarch-cp": "^0.10.1-pre.48",
     "open": "8.4.2",
-    "pdrive": "^0.0.23",
+    "pdrive": "^0.0.24",
     "portfinder": "^1.0.32",
     "qrcode": "^1.5.3",
     "random-useragent": "^0.5.0",

package/server/index.js

@@ -166,6 +166,63 @@ class Server {
       }
     })
   }
+  async chrome(req, res, type) {
+    let name = req.params.name
+    let app_path = this.kernel.path("api", name, "pinokio.js")
+    let rawpath = "/api/" + name
+    let config  = (await this.kernel.loader.load(app_path)).resolved
+    if (config && config.menu) {
+      if (typeof config.menu === "function") {
+        if (config.menu.constructor.name === "AsyncFunction") {
+          config.menu = await config.menu(this.kernel)
+        } else {
+          config.menu = config.menu(this.kernel)
+        }
+      }
+
+      let uri = this.kernel.path("api")
+      await this.renderMenu(uri, name, config, [])
+
+    } else {
+      // if there is no menu, display all files
+      let p = this.kernel.path("api", name)
+      let files = await fs.promises.readdir(p, { withFileTypes: true })
+      files = files.filter((file) => {
+        return file.name.endsWith(".json") || file.name.endsWith(".js")
+      })
+      config = {
+        title: name, 
+        menu: files.map((file) => {
+          return {
+            text: file.name,
+            href: file.name
+          }
+        })
+      }
+      let uri = this.kernel.path("api")
+      await this.renderMenu(uri, name, config, [])
+    }
+    let platform = os.platform()
+    //res.render("browser", {
+    res.render("browser-experimental", {
+      port: this.port,
+      type,
+      platform,
+      running:this.kernel.api.running,
+      memory: this.kernel.memory,
+      sidebar: "/pinokio/sidebar/" + name,
+      name,
+      tabs: (this.tabs[name] || []),
+      config,
+//        sidebar_url: "/pinokio/sidebar/" + name,
+      home: req.originalUrl,
+//        paths,
+      theme: this.theme,
+      agent: this.agent,
+      src: "/_api/" + name,
+      rawpath,
+    })
+  }
   async render(req, res, pathComponents, meta) {
 
     
@@ -767,6 +824,7 @@ class Server {
           execUrl: "~" + req.originalUrl.replace(/^\/_api/, "\/api"),
           proxies: this.kernel.api.proxies[filepath],
         }
+        console.log("result", result)
 
         res.render(template, result)
       } else {
@@ -928,7 +986,6 @@ class Server {
 
 
         let index = 0
-        console.log({ items })
         for(let i=0; i<items.length; i++) {
           let item = items[i]
           let p = path.resolve(uri, item.name, "pinokio.js")
@@ -1127,6 +1184,7 @@ class Server {
 
 
   async renderMenu(uri, name, config, pathComponents) {
+    console.log("config.menu =" ,config.menu)
     if (config.menu) {
       for(let i=0; i<config.menu.length; i++) {
         let menuitem = config.menu[i]
@@ -1691,6 +1749,36 @@ class Server {
 //    }
     await this.startLogging()
 
+    // check version from this.store
+    let version = this.kernel.store.get("version")
+    // if the version is different from package.json version, run update logic
+    if (this.kernel.homedir) {
+      if (version === this.version.pinokiod) {
+        console.log("version up to date")
+      } else {
+      // update the py module if it's already installed
+        let installed = await this.kernel.bin.mod.py.installed()
+        console.log("py installed", installed)
+        if (installed) {
+          // update
+          console.log("update py")
+          await this.kernel.exec({
+            message: "git pull",
+            path: this.kernel.path("bin/py")
+          }, (e) => {
+            console.log(e)
+          })
+
+        }
+        // after updating, set the version
+        console.log("set the version", this.version.pinokiod)
+        this.kernel.store.set("version", this.version.pinokiod)
+        console.log("RESTART")
+        this.listening = true   // set this.listening = true so all http connections get reset when restarting
+        this.start(this.debug)
+      }
+    }
+
     
 
     //await this.configure()
@@ -1836,6 +1924,80 @@ class Server {
 //        })
 //      }
     })
+
+
+
+//    /*
+//
+//        Requirements
+//
+//        1. When entering /app/:name, it should open the app's page if it's running. If not running, start the app, and open the app's page at the end
+//
+//        /app/:name
+//
+//        - check the /api/:name folder
+//          - if it exists:
+//            - check pinokio.js for start attribute
+//              - if start exists, 
+//          - if it doesn't exist:
+//
+//        2. how about when i want to use the tab view?
+//
+//
+//        what if the browser view stays, but just add a "default" attribute for tabs, and select that by default
+//
+//
+//        what if every script after starting, checks for local.url, and if it exists, 
+//
+//    */
+//    this.app.get("/app/:name", async (req, res) => {
+//
+//      // 1. take req.params.name and resolve the folder name
+//      let name = req.params.name
+//      let app_path = this.kernel.path("api", name, "pinokio.js")
+//      // 2. open pinokio.js to check the start attribute
+//      let config  = (await this.kernel.loader.load(app_path)).resolved
+//      if (config && config.start) {
+//        if (typeof config.start === "function") {
+//          if (config.start.constructor.name === "AsyncFunction") {
+//            config.start = await config.start(this.kernel)
+//          } else {
+//            config.start = config.start(this.kernel)
+//          }
+//        }
+//        // 3. check if the start script is running
+//        let startScriptPath = this.kernel.path("api", name, config.start)
+//        if (this.kernel.running(startScriptPath)) {
+//          // 4.a. if running, route to the local.url for the start script
+//          let local = this.kernel.local(startScriptPath)
+//          if (local && local.url) {
+//            // if local.url exists, display that
+//            res.redirect(local.url)
+//          } else {
+//            // if local.url doesn't exist, display the app page
+//            res.redirect(`http://localhost:${this.port}/pinokio/browser/${req.params.name}`)
+//          }
+//        } else {
+//          // 4.b. if not running, start the script and render the loading screen
+//          //    => the loading screen will automatically reload after the script has started
+//          if (typeof config.start === 'object') {
+//            let url = config.start.url
+//            let params = config.start.params
+//            let queryStr = new URLSearchParams(params).toString()
+//            let u = `http://localhost:${this.port}/pinokio/launch/${req.params.name}/${url}?${queryStr}`
+//            console.log("u1", u)
+//            res.redirect(u)
+//          } else {
+//            let url = config.start
+//            let u = `http://localhost:${this.port}/pinokio/launch/${req.params.name}/${url}`
+//            //let u = `/api/${req.params.name}/${url}`
+//            console.log("u2", u)
+//            res.redirect(u)
+//          }
+//        }
+//      }
+//    })
+
     this.app.get("/script/:name", (req, res) => {
       if (req.params.name === "start") {
         res.json(this.startScripts)
@@ -1995,59 +2157,11 @@ class Server {
         res.redirect("/")
       }
     })
+    this.app.get("/pinokio/launch/:name", async (req, res) => {
+      this.chrome(req, res, "launch")
+    })
     this.app.get("/pinokio/browser/:name", async (req, res) => {
-      let name = req.params.name
-      let app_path = this.kernel.path("api", name, "pinokio.js")
-      let rawpath = "/api/" + name
-      let config  = (await this.kernel.loader.load(app_path)).resolved
-      if (config && config.menu) {
-        if (typeof config.menu === "function") {
-          if (config.menu.constructor.name === "AsyncFunction") {
-            config.menu = await config.menu(this.kernel)
-          } else {
-            config.menu = config.menu(this.kernel)
-          }
-        }
-
-        let uri = this.kernel.path("api")
-        await this.renderMenu(uri, name, config, [])
-
-      } else {
-        // if there is no menu, display all files
-        let p = this.kernel.path("api", name)
-        let files = await fs.promises.readdir(p, { withFileTypes: true })
-        files = files.filter((file) => {
-          return file.name.endsWith(".json") || file.name.endsWith(".js")
-        })
-        config = {
-          title: name, 
-          menu: files.map((file) => {
-            return {
-              text: file.name,
-              href: file.name
-            }
-          })
-        }
-        let uri = this.kernel.path("api")
-        await this.renderMenu(uri, name, config, [])
-      }
-      let platform = os.platform()
-      res.render("browser", {
-        platform,
-        running:this.kernel.api.running,
-        memory: this.kernel.memory,
-        sidebar: "/pinokio/sidebar/" + name,
-        name,
-        tabs: (this.tabs[name] || []),
-        config,
-//        sidebar_url: "/pinokio/sidebar/" + name,
-        home: req.originalUrl,
-//        paths,
-        theme: this.theme,
-        agent: this.agent,
-        src: "/_api/" + name,
-        rawpath,
-      })
+      this.chrome(req, res, "browser")
     })
     this.app.post("/pinokio/delete", async (req, res) => {
       try {

package/server/public/Socket.js

@@ -82,7 +82,6 @@ class Socket {
           this.ws.send(JSON.stringify(rpc))
         });
         this.ws.addEventListener('message', (message) => {
-//          console.log("message", message)
           const packet = JSON.parse(message.data);
           ondata(packet)
         });

package/server/public/style.css

@@ -52,7 +52,6 @@
 }
 .navheader {
   padding: 15px 10px;
-  background: var(--dark-bg);
 }
 body.columns footer {
 }
@@ -1432,10 +1431,8 @@ b {
 }
 body.dark #new-window {
   color: var(--dark-btn-color);
-  /*
-  color: white;
-  */
 }
+/*
 #new-window {
   cursor: pointer;
   padding: 10px;
@@ -1444,6 +1441,7 @@ body.dark #new-window {
   font-size: 18px;
   color: var(--light-color);
 }
+*/
 body.dark .btn2 {
   color: var(--dark-btn-color);
   /*