pinokiod 3.19.2 → 3.19.4

package/kernel/util.js

@@ -1,5 +1,7 @@
 const fs = require('fs')
+const { spawn } = require('child_process')
 const http = require('http');
+const notifier = require('node-notifier');
 const os = require('os')
 const net = require('node:net')
 const path = require('path')
@@ -19,6 +21,7 @@ const {
 } = require('glob')
 
 
+  const platform = os.platform()
 // asar handling for go-get-folder-size
 let g
 if( __dirname.includes(".asar") ) {
@@ -36,6 +39,51 @@ const du = async (folderpath) => {
   return totalSize;
 }
 
+const filepicker = async(req, ondata, kernel) => {
+  if (req.params.filetype) {
+    /*
+      2 types:
+        filetype: [ 'images/*.png,*.jpg', 'docs/*.pdf' ]
+        filetype: 'images/*.png,*.jpg', 'docs/*.pdf'
+    */
+    let filetype
+    if (Array.isArray(req.params.filetype)) {
+      filetype = req.params.filetype
+    } else {
+      filetype = [req.params.filetype]
+    }
+    req.params.filetypes = filetype.map((str) => {
+      let chunks = str.split("/")
+      let type = chunks[0]
+      let extensions = chunks[1].split(",").join(" ")
+      return [type, extensions]
+    })
+  }
+  console.log("Filetype", req.params.filetype)
+  console.log("Filetypes", req.params.filetypes)
+  let response = await new Promise((resolve, reject) => {
+    let picker_path = kernel.path("bin/py/picker.py")
+    const proc = spawn("python", [picker_path])
+
+    let output = "";
+    proc.stdout.on("data", (chunk) => output += chunk);
+    proc.stderr.on("data", (err) => console.error("Python error:", err.toString()));
+    proc.on("close", () => {
+      try {
+        const result = JSON.parse(output);
+        if (result.error) return reject(result.error);
+        resolve(result.paths); // Always an array
+      } catch (e) {
+        reject("Failed to parse Python output: " + output);
+      }
+    });
+
+    proc.stdin.write(JSON.stringify(req.params));
+    proc.stdin.end();
+  });
+  return { paths: response }
+}
+
 const is_port_available = async (port) => {
   return new Promise((resolve) => {
     const server = http.createServer();
@@ -91,6 +139,33 @@ const parse_env = async (filename) => {
     return {}
   }
 }
+
+const exists= (abspath) => {
+  return new Promise(r=>fs.access(abspath, fs.constants.F_OK, e => r(!e)))
+}
+const log = async (filepath, str, session) => {
+  try {
+    await fs.promises.mkdir(filepath, { recursive: true })
+  } catch (e) {
+  }
+
+  let output = '';
+  for (let line of str.split('\n')) {
+    line = line.split('\r').pop(); // handle overwriting lines
+    output += line + '\n';
+  }
+
+  const pattern = [
+    '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]+)*|[a-zA-Z\\d]+(?:;[-a-zA-Z\\d\\/#&.:=?%@~_]*)*)?\\u0007)',
+    '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-Za-z=><~]))'
+  ].join('|');
+  const regex = new RegExp(pattern, 'gi')
+  let stripped = str.replaceAll(regex, '');
+  let logpath = path.resolve(filepath, session)
+  await fs.promises.writeFile(logpath, stripped)
+  let latest_logpath = path.resolve(filepath, "latest")
+  await fs.promises.writeFile(latest_logpath, stripped)
+}
 const run = (cmd, cwd, kernel) => {
 //  console.log("Util.run", { cmd, cwd })
 //  child_process.exec(cmd, { cwd })
@@ -373,7 +448,189 @@ function fill_object(obj, pattern, list, cache) {
   let res = recurse(obj);
   return { result: res, replaced_map }
 }
+function push(params) {
+  /*
+  params :- {
+    title: <string>,
+    subtitle: <string>, 
+    message: <string>,
+    image: <image path>,
+    sound: true|false,
+  }
+  */
+  if (params.image && !params.contentImage) {
+    params.contentImage = params.image
+  }
+  notifier.notify(params)
+}
+function p2u(localPath) {
+  /*
+    unix-like: /users/b/c => users/b/c
+    windows: C:\\\users\b\c => c/users/b/c
+  */
+  if (platform === 'win32') {
+    const match = localPath.match(/^([a-zA-Z]):\\(.*)/);
+    const drive = match[1].toLowerCase();
+    const path = match[2].replace(/\\/g, '/');
+    return `/${drive}/${path}`;
+  } else {
+    return `${localPath}`;
+  }
+}
+function u2p(urlPath) {
+  /*
+    unix-like: users/b/c => /users/b/c/
+    windows: c/users/b/c => C:\\users\b\c
+  */
+  const parts = urlPath.split('/');
+  if (platform === 'win32') {
+    const drive = parts[0];
+    return `${drive.toUpperCase()}:\\${parts.slice(1).join("\\")}`
+  } else {
+    return `/${parts.join("/")}`
+  }
+}
+
+function classifyChange(head, workdir, stage) {
+  const key = `${head}${workdir}${stage}`;
+
+  // HEAD | WORKDIR | STAGE
+  switch (key) {
+    case '000':
+      return 'unmodified'; // shouldn't show up
+    case '001':
+      return 'added (staged)';
+    case '002':
+      return 'added (unstaged)';
+    case '003':
+      return 'added (staged + modified)';
+    case '010':
+    case '020':
+      return 'untracked';
+    case '100':
+      return 'deleted (unstaged)';
+    case '101':
+      return 'deleted (staged)';
+    case '102':
+      return 'deleted (staged, but modified)';
+    case '110':
+      return 'modified (unstaged)';
+    case '111':
+      return 'modified (staged)';
+    case '112':
+      return 'modified (staged + unstaged)';
+    case '120':
+      return 'type changed (unstaged)';
+    case '121':
+      return 'type changed (staged)';
+    default:
+      return `unknown (${key})`;
+  }
+
+}
+
+
+function diffLinesWithContext(diffs, context = 3) {
+  const result = [];
+  let lineOld = 1;
+  let lineNew = 1;
+
+  const blocks = [];
+  let i = 0;
+
+  while (i < diffs.length) {
+    if (!diffs[i].added && !diffs[i].removed) {
+      // Flatten unchanged lines
+      const lines = [];
+      while (i < diffs.length && !diffs[i].added && !diffs[i].removed) {
+        const split = diffs[i].value.split('\n');
+        if (split[split.length - 1] === '') split.pop();
+        for (const line of split) {
+          lines.push({
+            line,
+            lineOld,
+            lineNew,
+          });
+          lineOld++;
+          lineNew++;
+        }
+        i++;
+      }
+      blocks.push({ type: 'context', lines });
+    } else {
+      // Change block (added or removed)
+      const change = [];
+      while (i < diffs.length && (diffs[i].added || diffs[i].removed)) {
+        const split = diffs[i].value.split('\n');
+        if (split[split.length - 1] === '') split.pop();
+        const type = diffs[i].added ? 'add' : 'del';
+
+        for (const line of split) {
+          change.push({
+            line,
+            lineOld: type === 'add' ? '' : lineOld,
+            lineNew: type === 'del' ? '' : lineNew,
+            type,
+          });
+          if (type !== 'add') lineOld++;
+          if (type !== 'del') lineNew++;
+        }
+        i++;
+      }
+      blocks.push({ type: 'change', lines: change });
+    }
+  }
+
+  // Now emit with context
+  const summarized = [];
+
+  for (let j = 0; j < blocks.length; j++) {
+    const block = blocks[j];
+
+    if (block.type === 'change') {
+      // Add leading context
+      const leading = (j > 0 && blocks[j - 1].type === 'context')
+        ? blocks[j - 1].lines.slice(-context)
+        : [];
 
+      const trailing = (j < blocks.length - 1 && blocks[j + 1].type === 'context')
+        ? blocks[j + 1].lines.slice(0, context)
+        : [];
+
+      // Push leading context
+      for (const l of leading) {
+        summarized.push({
+          line: l.line,
+          lineOld: l.lineOld,
+          lineNew: l.lineNew,
+          type: 'context',
+        });
+      }
+
+      // Push change lines
+      for (const l of block.lines) {
+        summarized.push(l);
+      }
+
+      // Push trailing context
+      for (const l of trailing) {
+        summarized.push({
+          line: l.line,
+          lineOld: l.lineOld,
+          lineNew: l.lineNew,
+          type: 'context',
+        });
+      }
+
+      // Remove consumed context from next context block
+      if (j < blocks.length - 1 && blocks[j + 1].type === 'context') {
+        blocks[j + 1].lines = blocks[j + 1].lines.slice(context);
+      }
+    }
+  }
+
+  return summarized;
+}
 module.exports = {
-  parse_env, log_path, api_path, update_env, parse_env_detail, openfs, port_running, du, is_port_available, find_python, find_venv, fill_object, run, openURL
+  parse_env, log_path, api_path, update_env, parse_env_detail, openfs, port_running, du, is_port_available, find_python, find_venv, fill_object, run, openURL, u2p, p2u, log, diffLinesWithContext, classifyChange, push, filepicker
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pinokiod",
-  "version": "3.19.2",
+  "version": "3.19.4",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -21,6 +21,7 @@
     "cross-fetch": "^3.1.5",
     "csv-parse": "^5.3.10",
     "decompress": "^4.2.1",
+    "diff": "^8.0.2",
     "dotenv": "^16.4.5",
     "ejs": "^3.1.9",
     "eol": "^0.9.1",
@@ -39,6 +40,7 @@
     "http-terminator": "^3.2.0",
     "https": "^1.0.0",
     "ini": "^5.0.0",
+    "isbinaryfile": "^5.0.4",
     "isomorphic-git": "^1.23.0",
     "jimini": "^0.0.8",
     "jsdom": "^24.0.0",
@@ -51,8 +53,9 @@
     "multer": "^1.4.5-lts.1",
     "node-downloader-helper": "^2.1.9",
     "node-gradio-client": "^0.14.6",
+    "node-notifier": "^10.0.1",
     "p-limit": "^3.1.0",
-    "pdrive": "^0.0.26",
+    "pdrive": "^0.0.27",
     "portfinder-cp": "^1.0.34",
     "proxy-agent": "^6.5.0",
     "qrcode": "^1.5.3",

package/server/index.js

@@ -1,5 +1,8 @@
 const express = require('express');
+const diff = require('diff')
 const kill = require('kill-sync')
+const { isBinaryFile } = require("isbinaryfile");
+const { glob, sync, hasMagic } = require('glob-gitignore')
 const portfinder = require('portfinder-cp');
 const proxy = require('express-http-proxy-cp');
 const sudo = require("sudo-prompt-programfiles-x86");
@@ -50,6 +53,11 @@ const Info = require("../kernel/info")
 
 const Setup = require("../kernel/bin/setup")
 
+function normalize(str) {
+  if (!str) return '';
+  return (str.endsWith('\n') ? str : str + '\n').replace(/\r\n/g, '\n');
+}
+
 class Server {
   constructor(config) {
     this.tabs = {}
@@ -358,7 +366,10 @@ class Server {
   }
 
   async chrome(req, res, type) {
+    let d = "Request " + Date.now()
+    console.time(d)
 
+    console.time("bin check " + d)
     let { requirements, install_required, requirements_pending, error } = await this.kernel.bin.check({
       bin: this.kernel.bin.preset("dev"),
     })
@@ -366,6 +377,7 @@ class Server {
       res.redirect(`/setup/dev?callback=${req.originalUrl}`)
       return
     }
+    console.timeEnd("bin check " + d)
 
     let name = req.params.name
     let config = await this.kernel.api.meta(name)
@@ -514,7 +526,11 @@ class Server {
       feed = this.newsfeed(gitRemote)
     }
 
+    // git
 
+    let c = this.kernel.path("api", name)
+
+    let repos = await this.kernel.git.repos(c)
     await this.kernel.plugin.init()
     let plugin = await this.getPlugin(name)
     let plugin_menu = null
@@ -527,6 +543,7 @@ class Server {
     let current_urls = await this.current_urls(req.originalUrl.slice(1))
 
     const result = {
+      repos,
       current_urls,
       path: this.kernel.path("api", name),
       plugin_menu,
@@ -562,6 +579,7 @@ class Server {
     if (!this.kernel.proto.config) {
       await this.kernel.proto.init()
     }
+    console.timeEnd(d)
     res.render("app", result)
   }
   getVariationUrls(req) {
@@ -585,7 +603,7 @@ class Server {
     if (rendered.id) {
       shell_id = encodeURIComponent(`${name}_${rendered.id}`)
     } else {
-      shell_id = encodeURIComponent(`${name}_${i}_${hash}`)
+      shell_id = encodeURIComponent(`${name}_${i}_session_${hash}`)
     }
     return shell_id
   }
@@ -947,7 +965,6 @@ class Server {
           bin: this.kernel.bin.preset("ai"),
           script: resolved
         })
-        console.log({ requirements })
 
         //let requirements = this.kernel.bin.requirements(resolved)
         //let requirements_pending = !this.kernel.bin.installed_initialized
@@ -988,7 +1005,6 @@ class Server {
 
 
         let env_requirements = await Environment.requirements(resolved, filepath, this.kernel)
-        console.log({ env_requirements })
         if (env_requirements.requires_instantiation) {
           let p = Util.api_path(filepath, this.kernel)
           let platform = os.platform()
@@ -1776,7 +1792,6 @@ class Server {
           }
           let shell_id = this.get_shell_id(name, currentIndexPath, rendered)
 
-
 //          let hash = crypto.createHash('md5').update(JSON.stringify(rendered)).digest('hex')
 //          let shell_id
 //          if (rendered.id) {
@@ -1789,26 +1804,36 @@ class Server {
           let shell = this.kernel.shell.get(shell_id)
           if (shell) {
             menuitem.running = true
+          } else {
+            let decoded_shell_id = decodeURIComponent(shell_id)
+            let shell = this.kernel.shell.get(decoded_shell_id)
+            if (shell) {
+              menuitem.running = true
+            }
           }
         }
 
         if (menuitem.href) {
           let u
+          let cwd
           if (menuitem.href.startsWith("http")) {
             menuitem.src = menuitem.href
           } else if (menuitem.href.startsWith("/")) {
             let run_path = "/run"
             if (menuitem.href.startsWith(run_path)) {
               u = new URL("http://localhost" + menuitem.href.slice(run_path.length))
+              cwd = u.searchParams.get("cwd")
               u.search = ""
               menuitem.src = u.pathname
             } else {
               u = new URL("http://localhost" + menuitem.href)
+              cwd = u.searchParams.get("cwd")
               u.search = ""
               menuitem.src = u.pathname
             }
           } else {
             u = new URL("http://localhost/" + menuitem.href)
+            cwd = u.searchParams.get("cwd")
             u.search = ""
             menuitem.src = u.pathname
           }
@@ -1829,7 +1854,6 @@ class Server {
               menuitem.running = true
             }
           }
-          
 
         }
 
@@ -2407,6 +2431,42 @@ class Server {
       return true
     }
   }
+  async getPluginGlobal(filepath) {
+    if (this.kernel.plugin.config) {
+      try {
+        let info = new Info(this.kernel)
+        info.cwd = () => {
+          return filepath
+        }
+        let menu = await this.kernel.plugin.config.menu(this.kernel, info)
+        let plugin = { menu }
+        let uri = filepath
+        await this.renderMenu(uri, filepath, plugin, [])
+
+        function setOnlineIfRunning(obj) {
+          if (Array.isArray(obj)) {
+            for (const item of obj) setOnlineIfRunning(item);
+          } else if (obj && typeof obj === 'object') {
+            if (obj.running === true) obj.online = true;
+            for (const key in obj) setOnlineIfRunning(obj[key]);
+          }
+        }
+
+        setOnlineIfRunning(plugin)
+
+        return plugin
+      } catch (e) {
+        console.log("getPlugin ERROR", e)
+        return {
+          menu: []
+        }
+      }
+    } else {
+      return {
+        menu: []
+      }
+    }
+  }
   async getPlugin(name) {
     console.log("getPlugin", name)
     if (this.kernel.plugin.config) {
@@ -3098,6 +3158,14 @@ class Server {
         res.json({ error: e.message })
       }
     }))
+    this.app.post("/push", ex(async (req, res) => {
+      try {
+        Util.push(req.body)
+        res.json({ success: true })
+      } catch (e) {
+        res.json({ error: e.stack })
+      }
+    }))
     this.app.post("/runcmd", ex(async (req, res) => {
       //Util.openfs(req.body.path, req.body.mode)
       let cwd = req.body.cwd
@@ -4304,6 +4372,129 @@ class Server {
         res.json(this.startScripts)
       }
     }))
+    this.app.get("/git/*", ex(async (req, res) => {
+      let { requirements, install_required, requirements_pending, error } = await this.kernel.bin.check({
+        bin: this.kernel.bin.preset("dev"),
+      })
+      if (!requirements_pending && install_required) {
+        res.redirect(`/setup/dev?callback=${req.originalUrl}`)
+        return
+      }
+      let dir = this.kernel.path("api", req.params[0])
+      let config = await this.kernel.git.config(dir)
+
+      let remote = null
+      if (config["remote \"origin\""]) {
+        remote = config["remote \"origin\""].url
+      }
+      let changes = []
+      try {
+        const statusMatrix = await git.statusMatrix({ dir, fs });
+        for (const [filepath, head, workdir, stage] of statusMatrix) {
+          if (head !== workdir || head !== stage) {
+            const fullPath = path.join(dir, filepath);
+            let relpath = path.relative(this.kernel.homedir, fullPath)
+            let webpath = "/asset/" + relpath
+
+            // Skip if binary
+            let binary = false;
+            try {
+              binary = await isBinaryFile(fullPath);
+            } catch {
+              binary = false; // fallback
+            }
+
+            if (binary) {
+              changes.push({
+                webpath,
+                file: filepath,
+                //status: `${head}${workdir}${stage}`,
+                status: Util.classifyChange(head, workdir, stage),
+                binary: true,
+                diff: null,
+              });
+              continue;
+
+            } else {
+
+              let oldContent = "";
+              let newContent = "";
+
+              // HEAD version (from Git blob)
+              try {
+                const commitOid = await git.resolveRef({ fs, dir, ref: "HEAD" });
+                const { blob } = await git.readBlob({ fs, dir, oid: commitOid, filepath });
+                oldContent = Buffer.from(blob).toString("utf8");
+              } catch (e) {
+                oldContent = "";
+              }
+
+              // Working directory version
+              try {
+                newContent = await fs.promises.readFile(fullPath, "utf8");
+              } catch (e) {
+                newContent = "";
+              }
+
+
+              const diffs = diff.diffLines(normalize(oldContent), normalize(newContent));
+
+              const summarized = Util.diffLinesWithContext(diffs, 5);
+              changes.push({ webpath, file: filepath, status: Util.classifyChange(head, workdir, stage)/*`${head}${workdir}${stage}`*/, diff: summarized, binary: false, });
+            }
+          }
+        }
+
+      } catch (err) {
+        console.log("git status matrix error", err)
+      }
+
+      res.render("git", {
+        changes,
+        dir,
+        config,
+        remote,
+        theme: this.theme,
+        platform: this.kernel.platform,
+        agent: this.agent,
+      })
+    }))
+    this.app.get("/dev/*", ex(async (req, res) => {
+      let { requirements, install_required, requirements_pending, error } = await this.kernel.bin.check({
+        bin: this.kernel.bin.preset("dev"),
+      })
+      if (!requirements_pending && install_required) {
+        res.redirect(`/setup/dev?callback=${req.originalUrl}`)
+        return
+      }
+      let platform = os.platform()
+      await this.kernel.plugin.init()
+      let filepath = Util.u2p(req.params[0])
+      let plugin = await this.getPluginGlobal(filepath)
+      let current_urls = await this.current_urls(req.originalUrl.slice(1))
+      let plugin_menu
+      try {
+        plugin_menu = plugin.menu[0].menu
+      } catch (e) {
+        plugin_menu = []
+      }
+      const result = {
+        current_urls,
+        plugin_menu,
+        portal: this.portal,
+        install: this.install,
+        port: this.port,
+        platform,
+        running:this.kernel.api.running,
+        memory: this.kernel.memory,
+        dynamic: "/pinokio/dynamic_global/" + req.params[0],
+        dynamic_content: null,
+        home: req.originalUrl,
+        theme: this.theme,
+        agent: this.agent,
+      }
+      res.render("mini", result)
+    }))
     this.app.get("/asset/*", ex((req, res) => {
       let pathComponents = req.params[0].split("/")
       let filepath = this.kernel.path(...pathComponents)
@@ -4384,6 +4575,24 @@ class Server {
         }
       }
     }))
+    this.app.get("/pinokio/dynamic_global/*", ex(async (req, res) => {
+      let plugin = await this.getPluginGlobal("/" + req.params[0])
+      let html = ""
+      if (plugin && plugin.menu) {
+        let plugin_menu
+        try {
+          plugin_menu = plugin.menu[0].menu
+        } catch (e) {
+          plugin_menu = []
+        }
+        html = await new Promise((resolve, reject) => {
+          ejs.renderFile(path.resolve(__dirname, "views/partials/dynamic.ejs"), { dynamic: plugin_menu }, (err, html) => {
+            resolve(html)
+          })
+        })
+      }
+      res.send(html)
+    }))
     this.app.get("/pinokio/dynamic/:name", ex(async (req, res) => {
   //    await this.kernel.plugin.init()
       let plugin = await this.getPlugin(req.params.name)

package/server/public/install.js

@@ -62,6 +62,10 @@ const install = async (name, url, term, socket, options) => {
     }
     socket.run({
       method: "shell.run",
+      client: {
+        cols: term.cols,
+        rows: term.rows,
+      },
       params: {
         message: cmd,
         path: "~/api"