pinokiod 1.2.25 → 1.2.27

package/kernel/api/index.js

@@ -40,7 +40,6 @@ class Api {
     this.checkProxy({ uri, name })
     // if the name exists, throw error
     // if the uri exists, throw error
-    console.log("startProxy", { scriptPath, uri, proxies: this.proxies })
     let proxy_uri = await this.lproxy.start(uri, options)
     if (this.proxies[scriptPath]) {
       this.proxies[scriptPath].push({ target: uri, proxy: proxy_uri, name})
@@ -53,7 +52,6 @@ class Api {
     }
   }
   stopProxy(config) {
-    console.log("stopProxy", { config, proxies: this.proxies })
     if (config.script) {
       // stop all proxies for the script
       if (this.proxies[config.script]) {
@@ -76,10 +74,8 @@ class Api {
         })
       }
     }
-    console.log("stopped", { proxies: this.proxies })
   }
   async init() {
-    console.log("api.init", { homedir: this.kernel.homedir })
     if (this.kernel.homedir) {
       this.userdir = path.resolve(this.kernel.homedir, "api")
       await fs.promises.mkdir(this.userdir, { recusrive: true, }).catch((e) => { })
@@ -88,10 +84,14 @@ class Api {
   }
   respond(req) {
     let requestPath = this.filePath(req.uri)
-    this.waiter[requestPath].resolve(req.response)
-    delete this.waiter[requestPath]
+    console.log("RESPOND", { req, waiter: this.waiter, requestPath })
+    if (this.waiter[requestPath]) {
+      this.waiter[requestPath].resolve(req.response)
+      delete this.waiter[requestPath]
+    }
   }
   wait(scriptPath) {
+    console.log("WAIT", scriptPath)
     // need to reject if everything fails
     return new Promise((resolve, reject) => {
       this.waiter[scriptPath] = { resolve, reject }
@@ -102,7 +102,6 @@ class Api {
     return result
   }
   async stop(req, ondata) {
-    console.log("STOP")
     // 1. set the "stop" flag for the uri, so the next execution in the queue for the uri will NOT queue another task
     // 2. stream a message closing the socket
 
@@ -204,7 +203,6 @@ class Api {
         }
       }
     }
-    console.log("gitPath", this.gitPath)
   }
   denormalize(gitRemote) {
     let denormalized
@@ -275,6 +273,19 @@ class Api {
       return null
     }
   }
+  getGitURI(uri) {
+    // git url
+    // test to see if any of the gitPaths match partially
+    let chunks = uri.split("/")
+    let collect = []
+    for(let chunk of chunks) {
+      collect.push(chunk) 
+      if (chunk.endsWith(".git")) {
+        break
+      }
+    }
+    return collect.join("/")
+  }
 
   resolveGitURI(uri) {
     // git url
@@ -474,9 +485,9 @@ class Api {
     return mod
 
   }
-  async step (request, rawrpc, input, i, total) {
+  async step (request, rawrpc, input, i, total, args) {
 
-    console.log("STEP", { request, rawrpc, input, i, total })
+    await this.kernel.api.init()
 
     // clear global regular expression object RegExp.lastMatch (memory leak prevention)
     let r = /\s*/g.exec("");
@@ -485,6 +496,7 @@ class Api {
 
     let memory = {
       input,
+      args,
       global: (this.kernel.memory.global[request.path] || {}),
       local: (this.kernel.memory.local[request.path] || {}),
       key: this.kernel.memory.key,
@@ -527,14 +539,12 @@ class Api {
     // replace {{{ }}} with {{ }}
     rpc = this.kernel.template.flatten(rpc)
 
-
     // 6. rpc must have method names
     if (rpc.method) {
 
       // 7. resolve the rpc
       let resolved = await this.resolveMethod(rpc, cwd)
 
-
       // 8. the endpoint must exist
       if (!resolved.method) {
         this.ondata({
@@ -569,6 +579,8 @@ class Api {
 
         rpc.input = input
 
+        rpc.args = args
+
         if (i < script.run.length-1) {
           rpc.next = i+1
         } else {
@@ -576,11 +588,11 @@ class Api {
         }
 
         if (rpc.hasOwnProperty("when")) {
-          console.log("rpc.when", { rpc })
           // if rpc.when is false, don't run this and go to the next step
           if (!rpc.when) {
             if (typeof rpc.next === "undefined" || rpc.next === null) {
               // last call
+              console.log("SCRIPT", script)
               if (script.daemon) {
                 this.ondata({
                   id: request.path,
@@ -606,13 +618,12 @@ class Api {
                   }
                 })
               }
-              return
+              return { request, input: null, step: rpc.next, total: script.run.length, args }
             } else {
               // still ongoing
               let next_rpc = script.run[rpc.next]
-              console.log("NEXT RPC", next_rpc, i, rpc.next)
               if (next_rpc) {
-                return { request, rawrpc: next_rpc, input: null, step: rpc.next, total: script.run.length }
+                return { request, rawrpc: next_rpc, input: null, step: rpc.next, total: script.run.length, args }
               }
             }
           }
@@ -630,6 +641,7 @@ class Api {
           let result = await this.run(resolved.method, rpc, (stream, type) => {
             let m = {
               id: request.path,
+              caller: request.caller,
               type: (type ? type : "stream"),
               index: i,
               total,
@@ -637,6 +649,17 @@ class Api {
               rpc,
               rawrpc
             }
+
+//            if (["input", "modal"].includes(m.type)) {
+//              // if a message requires user feedback, do not modify id
+//            } else {
+//              // if a message does not require user feedback
+//              // if the current session has a "caller" (parent process),
+//              // set the id to request.caller so the terminal prints the logs
+//              if (request.caller) {
+//                m.id = request.caller
+//              }
+//            }
             this.ondata(m)
           })
 
@@ -738,8 +761,10 @@ class Api {
 
           if (typeof rpc.next === "undefined" || rpc.next === null) {
 
+
             // kill all connected shells
             // if not daemon
+            console.log("DEF")
             if (script.daemon) {
               this.ondata({
                 id: request.path,
@@ -749,6 +774,7 @@ class Api {
                   description: "All scripts finished running. Running in daemon mode..."
                 }
               })
+              return { request, input: result, step: rpc.next, total: script.run.length, args }
             } else {
               // no next rpc to execute. Finish
               this.kernel.memory.local[request.path] = {}
@@ -764,15 +790,17 @@ class Api {
                   uri: request.path
                 }
               })
+              return { request, input: result, step: rpc.next, total: script.run.length, args }
             }
 
           } else {
             // still going
             let next_rpc = script.run[rpc.next]
-            return { request, rawrpc: next_rpc, input: result, step: rpc.next, total: script.run.length }
+            return { request, rawrpc: next_rpc, input: result, step: rpc.next, total: script.run.length, args }
           }
 
         } catch (e) {
+          console.log("<>ERROR", e)
           this.ondata({
             id: request.path,
             type: "error",
@@ -813,11 +841,26 @@ class Api {
     this.listeners[name] = undefined
   }
   createQueue(queue_id, concurrency) {
-    this.queues[queue_id] = fastq.promise(async ({ request, rawrpc, input, step, total, cwd }) => {
+    this.queues[queue_id] = fastq.promise(async ({ request, rawrpc, input, step, total, cwd, args }) => {
       try {
-        let response  = await this.step(request, rawrpc, input, step, total)
+        console.log("Before step", { request, rawrpc })
+        let response  = await this.step(request, rawrpc, input, step, total, args)
+        console.log("After step", response)
         if (response) {
-          this.queue(response.request, response.rawrpc, response.input, response.step, response.total, cwd)
+          if (response.rawrpc) {
+            this.queue(response.request, response.rawrpc, response.input, response.step, response.total, cwd, args)
+          } else {
+            if (response.request.caller) {
+              if (this.done[response.request.path]) {
+                this.done[response.request.path]({
+                  global: (this.kernel.memory.global[response.request.path] || {}),
+                  local: (this.kernel.memory.local[response.request.path] || {}),
+                  //return: response.input
+                  input: response.input
+                })
+              }
+            }
+          }
         } else {
           if (this.done[request.path]) {
             this.done[request.path]({
@@ -831,7 +874,7 @@ class Api {
       }
     }, concurrency)
   }
-  queue(request, rawrpc, input, step, total, cwd) {
+  queue(request, rawrpc, input, step, total, cwd, args) {
 
 
     // 1. only run one queued task at a time (for each api method)
@@ -872,7 +915,8 @@ class Api {
       input,
       step,
       total,
-      cwd
+      cwd,
+      args
     })
     let queueSize = this.queues[queue_id].length()
 
@@ -927,19 +971,17 @@ class Api {
 
 //    request.uri = this.resolvePath(this.userdir, request.uri)
 
-    let keypath = path.resolve(this.kernel.homedir, "key.json")
-    this.kernel.keys = (await this.loader.load(keypath)).resolved
+//    let keypath = path.resolve(this.kernel.homedir, "key.json")
+//    this.kernel.keys = (await this.loader.load(keypath)).resolved
 
     if (request.uri){
       this.counter++;
       // API Call
 
       request.path = this.resolvePath(this.userdir, request.uri)
-      console.log("#PROCESS", { request, userdir: this.userdir })
       let { cwd, script } = await this.resolveScript(request.path)
       request.cwd = cwd
 
-
       if (!script) {
         this.ondata({
           id: request.path,
@@ -955,7 +997,7 @@ class Api {
 
           this.done[request.path] = done
 
-          this.queue(request, script.run[0], request.input, 0, script.run.length, cwd)
+          this.queue(request, script.run[0], request.input, 0, script.run.length, cwd, request.input)
 
         } else {
           this.ondata({
@@ -1024,7 +1066,6 @@ class Api {
 
     // get the core script
     let script = (await this.loader.load(scriptpath)).resolved
-    console.log("##### ", { scriptpath, script })
 
     // if the sccript is a function, instantiate first
     if (typeof script === "function") {

package/kernel/api/input/index.js

@@ -1,4 +1,5 @@
 module.exports = async (req, ondata, kernel) => {
+  console.log("INPUT", req)
   ondata(req.params, "input")
   let response = await kernel.api.wait(req.parent.path)
   return response

package/kernel/api/log/index.js

@@ -9,6 +9,7 @@ module.exports = async (req, ondata, kernel) => {
       }
     }
   */
+  console.log("LLOG", req)
   if (req.params) {
     if (typeof req.params.raw !== "undefined") {
       ondata({

package/kernel/api/script/index.js

@@ -0,0 +1,92 @@
+const path = require('path')
+class Script {
+  async run(req, ondata, kernel) {
+    /*
+      {
+        "method": "script.run",
+        "params": {
+          "hash": <git commit hash>,
+          "branch": <git branch>,
+          "latest": true|false,
+          "uri": <script uri>,
+          "params": {
+          }
+        }
+      }
+
+      => uri resolve
+        - relative path
+        - absolute path
+        - http path
+    */
+    let uri
+    if (req.params.uri.startsWith("http")) {
+      uri = req.params.uri
+      let modpath = kernel.api.resolveGitURI(uri)
+      const repo_uri = kernel.api.getGitURI(uri)
+      console.log("repo_uri", repo_uri)
+      const urlWithoutProtocol = repo_uri.replace(/^(https?:\/\/)/, '');
+      const folderName = urlWithoutProtocol.replace(/[^a-zA-Z0-9_\-]/g, '_');
+      if (modpath) {
+        let msg = `Path ${uri} already exists at ${modpath}. Continuing...\r\n`
+        ondata({ raw: msg })
+      } else {
+        let msg = `Path ${uri} does not exist. Cloning...\r\n`
+        ondata({ raw: msg })
+
+        // clone
+        await kernel.bin.sh({
+          message: `git clone ${repo_uri} ${folderName}`,
+          path: kernel.api.userdir
+        }, ondata)
+      }
+
+      // switch branch
+      if (req.params.branch) {
+        await kernel.bin.sh({
+          message: `git switch ${req.params.branch}`,
+          path: kernel.path("api", folderName)
+        }, ondata)
+      }
+      if (req.params.hash) {
+        await kernel.bin.sh({
+          message: `git switch --detach ${req.params.hash}`,
+          path: kernel.path("api", folderName)
+        }, ondata)
+      }
+      if (req.params.latest) {
+        await kernel.bin.sh({
+          message: `git pull`,
+          path: kernel.path("api", folderName)
+        }, ondata)
+      }
+
+      await kernel.api.init()
+    } else {
+      uri = path.resolve(req.cwd, req.params.uri)
+    }
+    let res = await new Promise((resolve, reject) => {
+      kernel.api.process({
+        uri,
+        input: req.params.params,
+        client: req.client,
+        caller: req.parent.path,
+      }, (r) => {
+        resolve(r.input)
+      })
+    })
+    console.log("** RESPONSE", res)
+    // need to call api.linkGit() so the git repositories list is up to date
+    return res
+  }
+  async return(req, ondata, kernel) {
+    /*
+      {
+        "method": "script.return",
+        "params": <RETURN OBJECT>
+      }
+    */
+    return req.params
+  }
+}
+module.exports = Script

package/kernel/index.js

@@ -266,7 +266,6 @@ class Kernel {
         try {
           await sysinfo.refresh()
           let info = sysinfo.info
-          console.log("sysinfo", info)
           this.template.update(info)
           this.sysinfo = info
         } catch (e) {

package/kernel/shell.js

@@ -345,7 +345,6 @@ class Shell {
       this.persistent = params.persistent
     }
     this.prompt_pattern = await this.prompt(params.path)
-    console.log("this.prompt_pattern", this.prompt_pattern)
     this.cb = cb
     let r = await this.exec(params)
     return r
@@ -374,42 +373,28 @@ class Shell {
 
       //let re = /([\r\n]+[^\r\n]+)(\1)/gs
       let re = /(.+)(\1)/gs
-      console.log("get prompt - pty.spawn", config)
       let term = pty.spawn(this.shell, this.args, config)
-      console.log("terminal spawned", term)
       let ready
-      console.log("create terminal for prompt")
       let vt = new Terminal({
         allowProposedApi: true
       })
       let vts = new SerializeAddon()
       vt.loadAddon(vts)
 
-      console.log("start a queue for prompt")
       let queue = fastq((data, cb) => {
-        console.log("incoming queue", { data })
         vt.write(data, () => {
-          console.log("written")
           let buf = vts.serialize()
-          console.log("serialized buf", buf)
           let re = /(.+)echo pinokio[\r\n]+pinokio[\r\n]+(\1)/gs
           const match = re.exec(buf)
-          console.log({ match })
           if (match && match.length > 0) {
             let stripped = this.stripAnsi(match[1])
-            console.log({ match, stripped })
             const p = stripped
               .replaceAll(/[\r\n]/g, "")
               .trim()
               .replaceAll(/[.*+?^${}()|[\]\\]/g, '\\$&');
-            console.log("matched path", p)
-            console.log("kill term")
             term.kill()
-            console.log("term killed")
             vt.dispose()
-            console.log("vt disposed")
             queue.killAndDrain()
-            console.log("queue killAndDrained")
             resolve(p)
           }
         })
@@ -419,16 +404,12 @@ class Shell {
         console.log("onExit", { result })
       })
       term.onData((data) => {
-        console.log("term.onData", { ready, data })
         if (ready) {
-          console.log("push to queue", { data })
           queue.push(data)
         } else {
           setTimeout(() => {
-            console.log("ready?", ready)
             if (!ready) {
               ready = true
-              console.log("write", `echo pinokio${os.EOL}echo pinokio${os.EOL}`)
               term.write(`echo pinokio${os.EOL}echo pinokio${os.EOL}`)
             }
           }, 500)
@@ -529,8 +510,6 @@ class Shell {
       conda_name = "base"
     }
 
-    console.log("CONDA STEP 1", { conda: params.conda, conda_path, conda_name, conda_python })
-
 
     // 2. conda_activation
     let conda_activation = []
@@ -592,7 +571,6 @@ class Shell {
       conda_activation = []
     }
 
-    console.log("CONDA STEP 2", { conda_activation })
 
 
     // Update env setting
@@ -813,7 +791,6 @@ ${cleaned}
         let termination_prompt_re = new RegExp(this.prompt_pattern + "[ \r\n]*$", "g")
         let line = cleaned.replaceAll(/[\r\n]/g, "")
         let test = line.match(termination_prompt_re)
-        console.log({ termination_prompt_re, cleaned, line, test })
         if (test) {
           let cache = cleaned
           let cached_msg = msg

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "1.2.25",
+  "version": "1.2.27",
   "description": "",
   "main": "index.js",
   "scripts": {

package/server/index.js

@@ -297,9 +297,6 @@ class Server {
         args: "-c conda-forge"
       }, {
         name: "py"
-      }, {
-        gpu: "nvidia",
-        name: "cuda"
       }]
       let platform = os.platform()
       if (platform === "win32") {
@@ -310,6 +307,11 @@ class Server {
           name: "vs"
         })
       }
+      if (this.kernel.gpu === "nvidia") {
+        requirements.push({
+          name: "cuda"
+        })
+      }
       let requirements_pending = !this.kernel.bin.installed_initialized
 
 
@@ -554,9 +556,6 @@ class Server {
           args: "-c conda-forge"
         }, {
           name: "py"
-        }, {
-          gpu: "nvidia",
-          name: "cuda"
         }]
         let platform = os.platform()
         if (platform === "win32") {
@@ -569,6 +568,11 @@ class Server {
             name: "brew"
           })
         }
+        if (this.kernel.gpu === "nvidia") {
+          requirements.push({
+            name: "cuda"
+          })
+        }
 //        if (platform === "linux") {
 //          requirements.push({
 //            name: "brew"
@@ -1067,7 +1071,11 @@ class Server {
         notRunning,
         readme,
         filepath,
-        items: items.map((x) => {
+        items: items
+//        .filter((x) => {
+//          return x.type !== "lib"
+//        })
+        .map((x) => {
           //let name = (x.name.startsWith("0x") ? Buffer.from(x.name.slice(2), "hex").toString() : x.name)
           let name
           let description
@@ -1156,8 +1164,6 @@ class Server {
           menuitem.menu = m.menu
         }
 
-        console.log("MENUITEM", menuitem)
-
         if (menuitem.href && !menuitem.href.startsWith("http")) {
 
           // href resolution
@@ -1189,7 +1195,6 @@ class Server {
 
           // check running
           let fullpath = this.kernel.path(menuitem.src.slice(1))
-          console.log("fullpath", fullpath, menuitem.src, this.kernel.api.running)
           if (this.kernel.api.running[fullpath]) {
             menuitem.running = true
           }
@@ -1205,7 +1210,6 @@ class Server {
               let link = p.split(/[\/\\]/).filter((x) => { return x }).join("/")
               let uri = "~/api/" + name + "/" + link
 
-              console.log("*uri", uri)
               config.menu[i].action.uri = uri
             }
           }
@@ -1266,8 +1270,6 @@ class Server {
             }
           } else if (menuitem.hasOwnProperty("text")) {
 
-            console.log("processing", menuitem)
-
             if (menuitem.hasOwnProperty("icon")) {
               menuitem.html = `<i class="${menuitem.icon}"></i> ${menuitem.text}` 
             } else {
@@ -1287,7 +1289,6 @@ class Server {
             }
           }
         }
-        console.log("menu item", config.menu[i])
         if (config.menu[i].popout) {
           config.menu[i].target = "_blank"
         } else {
@@ -1311,22 +1312,17 @@ class Server {
 
         
       
-      console.log("config", JSON.stringify(config,null,2))
-
 
       // get all proxies that belong to this repository
       let childProxies = []
-      console.log("this.kernel.api.proxies=", this.kernel.api.proxies)
       for(let scriptPath in this.kernel.api.proxies) {
         let proxies = this.kernel.api.proxies[scriptPath]
-        console.log("PROXIES", proxies)
         for(let proxy of proxies) {
           if (scriptPath.startsWith(this.kernel.path("api", name))) {
             childProxies.push(proxy) 
           }
         }
       }
-      console.log({ childProxies })
 
       let proxyMenu = []
       for(let proxy of childProxies) {
@@ -1336,7 +1332,6 @@ class Server {
           href: proxy.proxy
         })
       }
-      console.log({ proxyMenu })
       config.menu = proxyMenu.concat(config.menu)
 
 
@@ -1925,13 +1920,10 @@ class Server {
 
         let uri = this.kernel.path("api")
         await this.renderMenu(uri, name, config, [])
-
-        console.log("#CONFIG", JSON.stringify(config, null, 2))
       }
 
 
       ejs.renderFile(path.resolve(__dirname, "views/partials/menu.ejs"), { menu: config.menu }, (err, html) => {
-        console.log("html", html)
         res.send(html)
       })
 

package/server/public/Socket.js

@@ -84,6 +84,7 @@ class Socket {
         this.ws.addEventListener('message', (message) => {
 //          console.log("message", message)
           const packet = JSON.parse(message.data);
+          console.log("PACKET", packet)
           ondata(packet)
         });
         this.ws.addEventListener('close', () => {

package/server/socket.js

@@ -17,6 +17,7 @@ class Socket {
       });
       ws.on('message', async (message) => {
         const req = JSON.parse(message);
+        console.log("MESSAGE", req)
         if (req.response) {
           this.parent.kernel.api.respond(req)
         } else {
@@ -94,6 +95,8 @@ class Socket {
     this.subscriptions.get(id).add(ws);
   }
   trigger(e) {
+
+    // send to id session
     let id
     if (e.kernel) {
       id = e.id
@@ -108,16 +111,30 @@ class Socket {
         if (subscriber.readyState === WebSocket.OPEN) {
           delete e.rpc
           delete e.rawrpc
-//          if (e.data && e.data.cleaned) delete e.data.cleaned
-//          if (e.data && e.data.state) delete e.data.state
           subscriber.send(JSON.stringify(e))
         }
       });
-    } else {
-//      if (!this.buffer[id]) {
-//        this.buffer[id] = []
-//      }
-//      this.buffer[id].push(e)
+    }
+
+    // send to caller session
+    if (e.caller) {
+      let caller
+      if (e.kernel) {
+        caller = e.caller
+      } else {
+        caller = this.parent.kernel.api.filePath(e.caller)
+      }
+
+      const subscribers = this.subscriptions.get(caller) || new Set();
+      if (subscribers.size > 0) {
+        subscribers.forEach((subscriber) => {
+          if (subscriber.readyState === WebSocket.OPEN) {
+            delete e.rpc
+            delete e.rawrpc
+            subscriber.send(JSON.stringify(e))
+          }
+        });
+      }
     }
   }
 }

package/server/views/fullscreen_editor.ejs

@@ -232,6 +232,7 @@ document.addEventListener("DOMContentLoaded", async () => {
 //              </div>`
             }
           } else if (packet.type === "stream") {
+            console.log("STREAM", packet)
             if (packet.data.raw) {
               this.term.write(packet.data.raw)
             } else if (packet.data.json) {
@@ -245,6 +246,7 @@ document.addEventListener("DOMContentLoaded", async () => {
             document.querySelector(".run .starting").classList.add("hidden")
             document.querySelector(".run .stop").classList.remove("hidden")
           } else if (packet.type === 'disconnect') {
+            console.log("DISCONNECT", packet)
             refreshParent(packet)
             reloadMemory()
             this.term.write("\r\nDisconnected...\r\n")
@@ -297,7 +299,9 @@ document.addEventListener("DOMContentLoaded", async () => {
             }
             this.socket.respond({
               response: {},
-              uri: "~" + location.pathname,
+//              uri: "~" + location.pathname,
+              uri: packet.id
+//              uri: (packet.caller ? packet.caller : packet.id)
             })
           } else if (packet.type === 'wait') {
             await WaitModal(packet.data)
@@ -305,7 +309,9 @@ document.addEventListener("DOMContentLoaded", async () => {
             await SimpleModal(packet.data)
             this.socket.respond({
               response: {},
-              uri: "~" + location.pathname,
+              //uri: "~" + location.pathname,
+              uri: packet.id
+//              uri: (packet.caller ? packet.caller : packet.id)
             })
           } else if (packet.type === "input") {
             let params = packet.data
@@ -319,7 +325,9 @@ document.addEventListener("DOMContentLoaded", async () => {
             if (input) {
               this.socket.respond({
                 response: input,
-                uri: "~" + location.pathname,
+                uri: packet.id
+//                uri: "~" + location.pathname,
+//                uri: (packet.caller ? packet.caller : packet.id)
               })
             }
             /*
@@ -443,6 +451,7 @@ document.addEventListener("DOMContentLoaded", async () => {
             document.querySelector(".run .stop").classList.add("hidden")
 
           } else if (packet.type === "event") {
+            console.log("EVENT", packet)
             if (packet.data === "stop") {
               document.querySelector("footer").innerHTML = "<b>Ready</b>"
               setTimeout(() => {