pinokiod 1.3.60 → 1.3.62

package/kernel/api/cloudflare/index.js

@@ -12,6 +12,7 @@ class C {
 
       => cloudflared tunnel --url http://localhost:8080
     */
+    console.log("TUNNEL", req)
     const message = `cloudflared tunnel --url ${req.params.uri}`
     let params = {
       id: message,
@@ -25,6 +26,7 @@ class C {
     if (req.cwd) options.cwd = req.cwd
     if (req.parent && req.parent.path) options.group = req.parent.path
     let pattern = /(https:.+?trycloudflare\.com)/
+    console.log("params", params)
     let cloudflare_url = await new Promise((resolve, reject) => {
       kernel.shell.start(params, options, (e) => {
         ondata(e)

package/kernel/api/env/index.js

@@ -0,0 +1,22 @@
+const set = require("../set")
+const rm = require("../rm")
+const Environment = require('../../environment')
+const Util = require("../../util")
+class Env {
+  async set(req, ondata, kernel) {
+    /*
+      req := {
+        "method": "env.set",
+        "params": {
+          <key>: <val>,
+          <key>: <val>,
+        }
+      }
+    */
+    // write to current app folder's ENVIRONMENT
+
+    let api_path = await Util.api_path(req.parent.path, kernel)
+    await Util.update_env(api_path, req.params)
+  }
+}
+module.exports = Env

package/kernel/api/fs/index.js

@@ -129,11 +129,13 @@ class FS {
     ondata({ raw: "\r\ncreating a shared drive:\r\n" + JSON.stringify(req.params, null, 2).replace(/\n/g, "\r\n") })
 
 
-    let api_path = await Util.api_path(req.parent.path, kernel)
-    let current_env = await Environment.get(api_path)
+    // start with the proces.env => merge pinokio global ENVIRONMENT => merge app ENVIRONMENT
     let default_env = await Environment.get(kernel.homedir)
-    current_env = Object.assign(process.env, default_env, current_env)
+    let api_path = await Util.api_path(req.parent.path, kernel)
+    let api_env = await Environment.get(api_path)
+    let current_env = Object.assign(process.env, default_env, api_env)
 
+    // if PINOKIO_DRIVE environment variable is specified, use this custom value for the drive path instead of ~/pinokio/drive
     let env_drive_path = current_env.PINOKIO_DRIVE
     let drive_home
     if (env_drive_path) {

package/kernel/api/index.js

@@ -560,7 +560,6 @@ class Api {
     // 6. rpc must have method names
     if (rpc.method) {
 
-
       // 7. resolve the rpc
       let resolved = await this.resolveMethod(rpc, cwd)
 
@@ -1051,20 +1050,34 @@ class Api {
         }
       }
     } else if (request.method) {
-
-      if (request.params && request.params.path) {
-        if (request.cwd) {
-          console.log("request.cwd", request.cwd)
-          let rendered_cwd = this.kernel.template.render(request.cwd, { kernel: this.kernel }, { allow_undefined: true })
-          console.log("rendered_cwd", rendered_cwd)
-          request.params.path = this.resolvePath(rendered_cwd, request.params.path)
+      let pass = 0;
+      while(true) {
+        request = this.kernel.template.render(request, { kernel: this.kernel })
+        if (this.kernel.template.istemplate(request)) {
+          pass++;
+          if (pass >= 4) {
+            // only try 4 times
+            break;
+          } else {
+            continue;
+          }
         } else {
-          request.params.path = this.resolvePath(this.userdir, request.params.path)
+          break;
         }
       }
+      if (this.kernel.template.istemplate(request)) {
+        console.log("something wrong with the request", request)
+        return
+      }
+
+      // replace {{{ }}} with {{ }}
+      request = this.kernel.template.flatten(request)
+
+      if (request.params && request.params.path) {
+        request.params.path = this.resolvePath(this.userdir, request.params.path)
+      }
 
       // Kernel Call
-      console.log("REQUEST", request)
       let resolved = await this.resolveMethod(request)
       try {
         resolved.dirname = resolved.dirname

package/kernel/environment.js

@@ -66,6 +66,11 @@ const ENVS = [{
   val: (home) => {
     return path.resolve(home, "drive")
   }
+}, {
+  key: "PINOKIO_SCRIPT_DEFAULT",
+  val: (home) => {
+    return "true"
+  }
 }];
 const folders = ENVS.filter((env) => {
   return env.type === "cache_folder" || env.type === "folder"
@@ -141,6 +146,15 @@ const APP_ENV = () => {
     "#",
     "##########################################################################",
     "PINOKIO_SHARE_CLOUDFLARE=false",
+    "",
+    "##########################################################################",
+    "#",
+    "# PINOKIO_SCRIPT_DEFAULT",
+    "# If this variable is false, 'default': true menu items in pinokio.js",
+    "# will NOT automatically run",
+    "#",
+    "##########################################################################",
+    "PINOKIO_SCRIPT_DEFAULT=true",
   ]
   return items.join("\n")
 }
@@ -160,7 +174,6 @@ const init_folders = async (homedir) => {
     }
   }
 }
-
 const get = async (homedir) => {
   const env_path = path.resolve(homedir, "ENVIRONMENT")
   const current_env = await util.parse_env(env_path)

package/kernel/index.js

@@ -43,6 +43,24 @@ class Kernel {
     this.exposed = {}
     this.envs = {}
   }
+  /*
+    kernel.env() => return the system environment
+    kernel.env("api/stablediffusion") => returns the full environment object
+  */
+  async env(...args) {
+    let folderpath
+    if (args) {
+      folderpath = path.resolve(this.homedir, ...args)
+    } else {
+      folderpath = this.homedir
+    }
+    console.log({ args , folderpath,homedir: this.homedir })
+    let api_path = await Util.api_path(folderpath, this)
+    let current_env = await Environment.get(api_path)
+    let default_env = await Environment.get(this.homedir)
+    console.log({ api_path, current_env, default_env })
+    return Object.assign(process.env, default_env, current_env)
+  }
   resumeprocess(uri) {
     let proc = this.procs[uri]
     if (proc && proc.resolve) {

package/kernel/shell.js

@@ -71,6 +71,8 @@ class Shell {
     this.cols = params.cols ? params.cols : 100;
     this.rows = params.rows ? params.rows : 30;
 
+    console.log({ cols: this.cols, rows: this.rows })
+
     this.vt = new Terminal({
       allowProposedApi: true,
       cols: this.cols,
@@ -112,19 +114,22 @@ class Shell {
 
     this.env.CMAKE_OBJECT_PATH_MAX = 1024
 
-    console.log("THIS.env before", this.env)
-    let home_env = await Environment.get(this.kernel.homedir)
-    console.log("home_env before", home_env)
-    this.env = Object.assign(this.env, home_env)
-    console.log("home_env after", this.env)
+    // First override this.env with system env
+    let system_env = await Environment.get(this.kernel.homedir)
+    this.env = Object.assign(this.env, system_env)
+
+    // if the shell is running from a script file, the params.$parent will include the path to the parent script
+    // this means we need to apply app environment as well
     if (params.$parent) {
       let api_path = await Util.api_path(params.$parent.path, this.kernel)
+
+      // initialize folders
       await Environment.init_folders(api_path)
-      let current_env = await Environment.get(api_path)
-      console.log({ current_env, api_path })
-      this.env = Object.assign(this.env, current_env)
+
+      // apply app env to this.env
+      let app_env = await Environment.get(api_path)
+      this.env = Object.assign(this.env, app_env)
     }
-    console.log("THIS.env after", this.env)
     let PATH_KEY;
     if (this.env.Path) {
       PATH_KEY = "Path"

package/kernel/shells.js

@@ -160,7 +160,6 @@ class Shells {
                 matches[2] += "g"   // if g option is not included, include it (need it for matchAll)
               }
               let re = new RegExp(matches[1], matches[2])
-              console.log("stream", stream)
               let line = stream.cleaned.replaceAll(/[\r\n]/g, "")
               //let rendered_event = [...stream.cleaned.matchAll(re)]
               let rendered_event = [...line.matchAll(re)]
@@ -222,7 +221,6 @@ class Shells {
       for(let i=0; i<shells.length; i++) {
         let shell = shells[i]
         if (shell.id === request.id) {
-          console.log("FOUND SHELL TO KILL", shell.id)
           shell.kill(message, true)
         }
       }
@@ -235,7 +233,6 @@ class Shells {
       for(let i=0; i<shells.length; i++) {
         let shell = shells[i]
         if (shell.group === request.group) {
-          console.log("kill shell ", request.group, i, shell)
           shell.kill(message, true)
         }
       }

package/kernel/util.js

@@ -1,6 +1,17 @@
 const fs = require('fs')
 const path = require('path')
 const dotenv = require('dotenv')
+
+const {auto: normalizeEOL} = require("eol");
+const {EOL} = require("os");
+const breakPattern = /\n/g;
+const breakReplacement = "\\n";
+const groupPattern = /\$/g;
+const groupReplacement = "$$$";
+const h = "[^\\S\\r\\n]";  // simulate `\h`
+const returnPattern = /\r/g;
+const returnReplacement = "\\r";
+
 const parse_env = async (filename) => {
   try {
     const buf = await fs.promises.readFile(filename)
@@ -10,6 +21,52 @@ const parse_env = async (filename) => {
     return {}
   }
 }
+
+const parse_env_detail = async (filename) => {
+  // takes env and returns an array
+  const config = parse_env(filename)
+  /*
+    config := {
+      key1: val1,
+      key2; val2,
+      ...
+    }
+  */
+  const keys = Object.keys(config)
+  const str = await fs.promises.readFile(filename, "utf8")
+  const lines = str.split(EOL)
+
+  let items = []
+
+  for(let line of lines) {
+    if (line.trim().startsWith("#")) {
+      items.push({
+        type: "comment",
+        val: line
+      })
+    } else {
+      const buf = Buffer.from(line)
+      const parsed = dotenv.parse(buf)
+      const key = Object.keys(parsed)
+      if (key.length > 0) {
+        items.push({ key: key[0], val: parsed[key[0]] })
+      }
+    }
+  }
+  /*
+  items := [{
+    type: "comment",
+    val: "###################################",
+  }, {
+    type: "comment",
+    val: "# this is a comment",
+  }, {
+    key: "TMPDIR",
+    val: "/tmp"
+  }]
+  */
+  return items
+}
 const api_path = async (fullpath, kernel) => {
   let api_path = `${kernel.homedir}${path.sep}api`
   let rel_path = path.relative(api_path, fullpath)
@@ -17,6 +74,55 @@ const api_path = async (fullpath, kernel) => {
   let current_api_path = `${api_path}${path.sep}${api_name}`
   return current_api_path
 }
+const escapeStringRegexp = (string) => {
+	if (typeof string !== 'string') {
+		throw new TypeError('Expected a string');
+	}
+
+	// Escape characters with special meaning either inside or outside character sets.
+	// Use a simple backslash escape when it’s always valid, and a `\xnn` escape when the simpler form would be disallowed by Unicode patterns’ stricter grammar.
+	return string
+		.replace(/[|\\{}()[\]^$+*?.]/g, '\\$&')
+		.replace(/-/g, '\\x2d');
+}
+
+const update_env = async (filepath, changes) => {
+  const envString = await fs.promises.readFile(filepath, "utf8")
+  console.log({ filepath, envString, changes })
+	let hasAppended = false;
+  console.log("changes", changes)
+  const newval = Object.keys(changes).reduce((result, varname) => {
+		const value = changes[varname].toString()
+			.replace(breakPattern, breakReplacement)
+			.replace(returnPattern, returnReplacement)
+			.trim();
+		const safeName = escapeStringRegexp(varname);
+		const varPattern = new RegExp(`^(${h}*${safeName}${h}*=${h}*)\\S*(${h}*)$`, "gm");
+		if (varPattern.test(result)) {
+			const safeValue = value.replace(groupPattern, groupReplacement);
+			return result.replace(varPattern, `$1${safeValue}$2`);
+		} else if (result === "") {
+			hasAppended = true;
+			return `${varname}=${value}${EOL}`;
+		} else if (!result.endsWith(EOL) && !hasAppended) {
+			hasAppended = true;
+			// Add an extra break between previously defined and newly appended variable
+			return `${result}${EOL}${EOL}${varname}=${value}`;
+		} else if (!result.endsWith(EOL)) {
+			// Add break for appended variable
+			return `${result}${EOL}${varname}=${value}`;
+		} else if (result.endsWith(EOL) && !hasAppended) {
+			hasAppended = true;
+			// Add an extra break between previously defined and newly appended variable
+			return `${result}${EOL}${varname}=${value}${EOL}`;
+		} else /*if (result.endsWith(EOL))*/ {
+			// Add break for appended variable
+			return `${result}${varname}=${value}${EOL}`;
+		}
+	}, normalizeEOL(envString));
+  console.log("updated ENVIRONMENT", newval)
+  await fs.promises.writeFile(filepath, newval)
+};
 module.exports = {
-  parse_env, api_path
+  parse_env, api_path, update_env, parse_env_detail
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "1.3.60",
+  "version": "1.3.62",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -21,6 +21,7 @@
     "decompress": "^4.2.1",
     "dotenv": "^16.4.5",
     "ejs": "^3.1.9",
+    "eol": "^0.9.1",
     "express": "^4.18.2",
     "express-http-proxy-cp": "^2.0.3",
     "express-session": "^1.17.3",

package/server/index.js

@@ -36,6 +36,7 @@ const Kernel = require("../kernel")
 const packagejson = require("../package.json")
 const Environment = require("../kernel/environment")
 const Cloudflare = require("../kernel/api/cloudflare")
+const Util = require("../kernel/util")
 class Server {
   constructor(config) {
     this.tabs = {}
@@ -233,14 +234,26 @@ class Server {
     let current = this.kernel.path("api", name, "ENVIRONMENT")
       // if environment.json doesn't exist, 
     let exists = await this.exists(current)
+    console.log("exists", exists)
     if (!exists) {
       let content = await Environment.APP_ENV()
+      console.log("write", content)
       await fs.promises.writeFile(current, content)
     }
 
+    let mode = "run"
+    console.log("req.query", req.query)
+    if (req.query && req.query.mode) {
+      mode = req.query.mode
+    }
+
+    const env = await this.kernel.env("api/" + name)
+    console.log("env", env)
 
     //res.render("browser", {
     res.render("browser-experimental", {
+      env,
+      mode,
       port: this.port,
       mem,
       type,
@@ -1272,9 +1285,22 @@ class Server {
     if (config.menu) {
 
       config.menu = [{
+        base: "/",
         text: "Configure",
-        href: "ENVIRONMENT?mode=source",
+        href: `env/api/${name}/ENVIRONMENT`,
         icon: "fa-solid fa-gear"
+//      }, {
+//        base: "/",
+//        text: "Public Share",
+//        action: {
+//          method: "env.set",
+//          params: {
+//            "PINOKIO_SHARE_CLOUDFLARE": true,
+//            "PINOKIO_SHARE_LOCAL": true
+//          }
+//        },
+//        href: `env/api/${name}/ENVIRONMENT`,
+//        icon: "fa-solid fa-gear"
       }].concat(config.menu)
 
       for(let i=0; i<config.menu.length; i++) {
@@ -1285,14 +1311,18 @@ class Server {
           menuitem.menu = m.menu
         }
 
-        if (menuitem.href && !menuitem.href.startsWith("http")) {
-
-          // href resolution
-          let absolute = path.resolve(__dirname, ...pathComponents, menuitem.href)
-          let seed = path.resolve(__dirname)
-          let p = absolute.replace(seed, "")
-          let link = p.split(/[\/\\]/).filter((x) => { return x }).join("/")
-          config.menu[i].href = "/api/" + name + "/" + link
+        if (menuitem.base === "/") {
+          config.menu[i].href = menuitem.base + menuitem.href
+        } else {
+          if (menuitem.href && !menuitem.href.startsWith("http")) {
+
+            // href resolution
+            let absolute = path.resolve(__dirname, ...pathComponents, menuitem.href)
+            let seed = path.resolve(__dirname)
+            let p = absolute.replace(seed, "")
+            let link = p.split(/[\/\\]/).filter((x) => { return x }).join("/")
+            config.menu[i].href = "/api/" + name + "/" + link
+          }
         }
 
         if (menuitem.href && menuitem.params) {
@@ -2118,6 +2148,49 @@ class Server {
         res.json({ error: "type must be 'local' or 'cloudflare'" })
       }
     })
+    this.app.post("/env", async (req, res) => {
+      console.log("req.body", req.body)
+      let fullpath = path.resolve(this.kernel.homedir, req.body.filepath)
+      let updated = req.body.vals
+      await Util.update_env(fullpath, updated)
+      res.json({})
+    })
+    this.app.get("/env/*", async (req, res) => {
+      let pathComponents = req.params[0].split("/")
+      let filepath = path.resolve(this.kernel.homedir, req.params[0])
+      let editorpath = "/edit/" + req.params[0]
+      const items = await Util.parse_env_detail(filepath)
+      res.render("env_editor", {
+        editorpath,
+        items,
+        theme: this.theme,
+        filepath,
+        agent: this.agent,
+      })
+    })
+    this.app.get("/share/:name", async (req, res) => {
+      let filepath = path.resolve(this.kernel.homedir, "api", req.params.name, "ENVIRONMENT")
+      //let filepath = path.resolve(this.kernel.homedir, req.params[0])
+      console.log("filepath", filepath)
+      const config = await Util.parse_env(filepath)
+      console.log("config", config)
+      const keys = [
+        "PINOKIO_SHARE_CLOUDFLARE",
+        "PINOKIO_SHARE_LOCAL",
+      ]
+      for(let key of keys) {
+        if (!config[key]) {
+          config[key] = ""
+        }
+      }
+      res.render("share_editor", {
+        keys,
+        config,
+        theme: this.theme,
+        filepath,
+        agent: this.agent,
+      })
+    })
     this.app.get("/edit/*", async (req, res) => {
       let pathComponents = req.params[0].split("/")
       let filepath = path.resolve(this.kernel.homedir, req.params[0])
@@ -2292,8 +2365,13 @@ class Server {
     this.app.get("/pinokio/launch/:name", async (req, res) => {
       this.chrome(req, res, "launch")
     })
+    this.app.get("/pinokio/browser/:name/browse", async (req, res) => {
+      console.log("browse mode")
+      this.chrome(req, res, "browse")
+    })
     this.app.get("/pinokio/browser/:name", async (req, res) => {
-      this.chrome(req, res, "browser")
+      console.log("run mode")
+      this.chrome(req, res, "run")
     })
     this.app.post("/pinokio/delete", async (req, res) => {
       try {

package/server/public/style.css

@@ -21,7 +21,10 @@
   --light-thin: rgba(0,0,0,0.1);
   --dark-thin: rgba(255,255,255,0.03);
 
+/*
   --dark-color: rgba(255,255,255,0.8);
+  */
+  --dark-color: white;
   --light-color: rgba(0,0,0,0.9);
 
   --dark-fade: rgba(255,255,255,0.6);
@@ -894,13 +897,11 @@ body.dark .readme a {
 body.dark .line {
   border-bottom: 1px solid var(--dark-thin);
 }
-body.dark .line.selected {
+body.dark .line.selected, body.dark .line:hover {
 /*
   border-left: 6px solid white;
   */
   background: rgba(80, 100, 70, 0.1);
-  border-top: 1px solid rgba(80, 100, 70, 0.1);
-  border-bottom: 1px solid rgba(80, 100, 70, 0.1);
 }
 /*
 body.dark .line:hover {
@@ -910,13 +911,11 @@ body.dark .line:hover {
   border-left: 6px solid black;
 }
 */
-.line.selected {
+.line.selected, .line:hover {
 /*
   border-left: 6px solid black;
   */
   background: rgba(70, 200, 250, 0.1);
-  border-top: 1px solid rgba(70, 200, 250, 0.3);
-  border-bottom: 1px solid rgba(70, 200, 250, 0.3);
 }
 .line {
   text-decoration: none;
@@ -1554,7 +1553,7 @@ nav.error-message {
   color: #8fd400;
 }
 */
-.running-apps .spinner {
+.running-apps .spinner, .not-running-apps .spinner {
   width: 50px;
   text-align: center;
   font-size: 20px;

package/server/socket.js

@@ -106,9 +106,17 @@ class Socket {
       //const subscribers = this.subscriptions.get(e.id) || new Set();
       subscribers.forEach((subscriber) => {
         if (subscriber.readyState === WebSocket.OPEN) {
-          delete e.rpc
-          delete e.rawrpc
-          subscriber.send(JSON.stringify(e))
+          let res = Object.assign({}, e)
+          delete res.rpc
+          delete res.rawrpc
+          if (res.data && res.data.id && res.data.raw) {
+            res.data = {
+              id: res.data.id,
+              raw: res.data.raw
+            }
+          }
+          console.log("*** res1", res)
+          subscriber.send(JSON.stringify(res))
         }
       });
     }
@@ -126,9 +134,19 @@ class Socket {
       if (subscribers.size > 0) {
         subscribers.forEach((subscriber) => {
           if (subscriber.readyState === WebSocket.OPEN) {
-            delete e.rpc
-            delete e.rawrpc
-            subscriber.send(JSON.stringify(e))
+
+            let res = Object.assign({}, e)
+            delete res.rpc
+            delete res.rawrpc
+            if (res.data && res.data.id && res.data.raw) {
+              res.data = {
+                id: res.data.id,
+                raw: res.data.raw
+              }
+            }
+            console.log("*** res2", res)
+            subscriber.send(JSON.stringify(res))
+
           }
         });
       }