pinokiod 7.2.7 → 7.2.9

package/kernel/api/index.js

@@ -11,6 +11,7 @@ const fastq = require('fastq')
 const Loader = require("../loader")
 const Environment = require("../environment")
 const Util = require('../util')
+const ShellRunTemplate = require('./shell_run_template')
 
 class Api {
   constructor(kernel) {
@@ -416,6 +417,12 @@ class Api {
         await this.process(script.on.stop) 
       }
     }
+    if (this.kernel.watch && typeof this.kernel.watch.stop === "function") {
+      if (req.params.id) {
+        await this.kernel.watch.stop(req.params.id)
+      }
+      await this.kernel.watch.stop(requestPath)
+    }
     // reset modules
     let modpath = this.resolvePath(cwd, req.params.uri)
     if (this.child_procs[modpath]) {
@@ -1010,6 +1017,7 @@ class Api {
     }
     // replace {{{ }}} with {{ }}
     rpc = this.kernel.template.flatten(rpc)
+    rpc = ShellRunTemplate.renderEnvArgs(this.kernel, rpc, memory)
 
     // 6. rpc must have method names
     if (rpc.method) {
@@ -1454,6 +1462,25 @@ class Api {
     }
     return false
   }
+  async startWatchersForRequest(request, script, scriptDir, input) {
+    if (!this.kernel.watch || typeof this.kernel.watch.startForScript !== "function") {
+      return
+    }
+    const id = request.id || request.path
+    if (!id) {
+      return
+    }
+    const cwd = request.cwd || scriptDir
+    await this.kernel.watch.startForScript({
+      id,
+      request,
+      script,
+      cwd,
+      dirname: scriptDir,
+      input,
+      args: input
+    })
+  }
   createQueue(queue_id, concurrency) {
     this.queues[queue_id] = fastq.promise(async ({ request, rawrpc, input, step, total, cwd, args }) => {
       try {
@@ -1695,6 +1722,7 @@ class Api {
             }
 
             const initialPayload = typeof request.input === "undefined" ? {} : request.input
+            await this.startWatchersForRequest(request, script, cwd, initialPayload)
             this.queue(request, steps[0], initialPayload, 0, steps.length, cwd, initialPayload)
 
           } else {

package/kernel/api/shell_run_template.js

@@ -0,0 +1,273 @@
+function isCmdShellName(shellName) {
+  const name = (shellName || '').toLowerCase()
+  return name.includes('cmd.exe') || name === 'cmd'
+}
+
+function isPowerShellName(shellName) {
+  const name = (shellName || '').toLowerCase()
+  return name.includes('powershell') || name.includes('pwsh')
+}
+
+const ENV_ARG_MARKER_RE = /__PINOKIO_ENVARG_(\d+)__/g
+
+function envArgMarker(index) {
+  return `__PINOKIO_ENVARG_${index}__`
+}
+
+function isPinokioEnvArgKey(key) {
+  return /^PINOKIO_ARG_\d+$/.test(key || "")
+}
+
+function hasEnvArgMarker(value) {
+  ENV_ARG_MARKER_RE.lastIndex = 0
+  return ENV_ARG_MARKER_RE.test(String(value))
+}
+
+function quotePosixLiteral(value) {
+  const input = value == null ? "" : String(value)
+  return `'${input.split("'").join("'\"'\"'")}'`
+}
+
+function quotePowerShellComposite(value) {
+  const input = value == null ? "" : String(value)
+  ENV_ARG_MARKER_RE.lastIndex = 0
+  let output = '"'
+  let lastIndex = 0
+  for (const match of input.matchAll(ENV_ARG_MARKER_RE)) {
+    const literal = input.slice(lastIndex, match.index)
+    output += literal.replace(/[`"$]/g, (char) => "`" + char)
+    output += "${env:PINOKIO_ARG_" + match[1] + "}"
+    lastIndex = match.index + match[0].length
+  }
+  output += input.slice(lastIndex).replace(/[`"$]/g, (char) => "`" + char)
+  output += '"'
+  return output
+}
+
+function quoteCmdComposite(value) {
+  const input = value == null ? "" : String(value)
+  ENV_ARG_MARKER_RE.lastIndex = 0
+  let output = '"'
+  let lastIndex = 0
+  for (const match of input.matchAll(ENV_ARG_MARKER_RE)) {
+    const literal = input.slice(lastIndex, match.index)
+    output += literal.replace(/([()%!^"<>&|])/g, '^$1')
+    output += "!PINOKIO_ARG_" + match[1] + "!"
+    lastIndex = match.index + match[0].length
+  }
+  output += input.slice(lastIndex).replace(/([()%!^"<>&|])/g, '^$1')
+  output += '"'
+  return output
+}
+
+function quoteEnvArgComposite(value, shellName) {
+  const input = value == null ? "" : String(value)
+  if (isCmdShellName(shellName)) {
+    return quoteCmdComposite(input)
+  }
+  if (isPowerShellName(shellName)) {
+    return quotePowerShellComposite(input)
+  }
+
+  ENV_ARG_MARKER_RE.lastIndex = 0
+  const parts = []
+  let lastIndex = 0
+  for (const match of input.matchAll(ENV_ARG_MARKER_RE)) {
+    const literal = input.slice(lastIndex, match.index)
+    if (literal) {
+      parts.push(quotePosixLiteral(literal))
+    }
+    parts.push('"$PINOKIO_ARG_' + match[1] + '"')
+    lastIndex = match.index + match[0].length
+  }
+  const tail = input.slice(lastIndex)
+  if (tail) {
+    parts.push(quotePosixLiteral(tail))
+  }
+  return parts.length > 0 ? parts.join("") : "''"
+}
+
+function shellNameFor(kernel, params) {
+  let shellName = kernel && kernel.platform === "win32" ? "cmd.exe" : "bash"
+  if (params && typeof params.shell === "string" && params.shell.trim()) {
+    shellName = params.shell
+  }
+  return shellName
+}
+
+function isPlainObject(value) {
+  return value && value.constructor === Object
+}
+
+function hasMultiline(value) {
+  return typeof value === "string" && /[\r\n]/.test(value)
+}
+
+function isStructuredArgvMessage(value) {
+  return isPlainObject(value) && Array.isArray(value._)
+}
+
+function hasStructuredArgvMessage(value) {
+  if (isStructuredArgvMessage(value)) {
+    return true
+  }
+  if (Array.isArray(value)) {
+    return value.some((item) => isStructuredArgvMessage(item))
+  }
+  return false
+}
+
+function protectStructuredString(value, state) {
+  if (!hasMultiline(value)) {
+    return value
+  }
+  const name = `PINOKIO_ARG_${state.args.length}`
+  state.args.push({
+    name,
+    value: value == null ? "" : String(value)
+  })
+  return envArgMarker(state.args.length - 1)
+}
+
+function protectStructuredValue(value, state) {
+  if (typeof value === "string") {
+    return protectStructuredString(value, state)
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => protectStructuredValue(item, state))
+  }
+  if (isPlainObject(value)) {
+    const rendered = {}
+    for (const [key, item] of Object.entries(value)) {
+      rendered[key] = protectStructuredValue(item, state)
+    }
+    return rendered
+  }
+  return value
+}
+
+function protectStructuredMessage(value, state) {
+  if (isStructuredArgvMessage(value)) {
+    return protectStructuredValue(value, state)
+  }
+  if (Array.isArray(value)) {
+    return value.map((item) => isStructuredArgvMessage(item) ? protectStructuredValue(item, state) : item)
+  }
+  return value
+}
+
+function renderEnvArgs(kernel, rpc, memory) {
+  if (!rpc || rpc.method !== "shell.run" || !rpc.params || !hasStructuredArgvMessage(rpc.params.message)) {
+    return rpc
+  }
+
+  const shellName = shellNameFor(kernel, rpc.params)
+  const state = { args: [] }
+  const message = protectStructuredMessage(rpc.params.message, state)
+
+  if (state.args.length === 0) {
+    return rpc
+  }
+
+  const env = Object.assign({}, rpc.params.env || {})
+  for (const arg of state.args) {
+    env[arg.name] = arg.value
+  }
+
+  return {
+    ...rpc,
+    params: {
+      ...rpc.params,
+      message,
+      env,
+      _pinokio_env_args: state.args,
+      _pinokio_cmd_delayed_expansion: isCmdShellName(shellName)
+    }
+  }
+}
+
+function envArgDetails(value) {
+  const normalized = String(value == null ? "" : value)
+    .replace(/\r\n/g, "\n")
+    .replace(/\r/g, "\n")
+    .replace(/[\x00-\x08\x0B\x0C\x0E-\x1F\x7F]/g, "?")
+  const lines = normalized.split("\n")
+  const previewLines = []
+  const maxLines = 8
+  const maxChars = 800
+  let used = 0
+
+  for (const line of lines.slice(0, maxLines)) {
+    const remaining = maxChars - used
+    if (remaining <= 0) {
+      break
+    }
+    if (line.length > remaining) {
+      previewLines.push(line.slice(0, remaining) + "...")
+      used = maxChars
+      break
+    }
+    previewLines.push(line)
+    used += line.length + 1
+  }
+
+  const preview = previewLines.join("\n")
+  const truncated = lines.length > previewLines.length || normalized.length > preview.length
+  if (truncated && previewLines[previewLines.length - 1] !== "...") {
+    previewLines.push("...")
+  }
+
+  return {
+    lineCount: normalized.length === 0 ? 0 : lines.length,
+    previewLines,
+    truncated
+  }
+}
+
+function formatEnvArgsPreview(args) {
+  if (!Array.isArray(args) || args.length === 0) {
+    return ""
+  }
+  const lines = ["\r\nPinokio shell args", ""]
+  for (const arg of args) {
+    const details = envArgDetails(arg.value)
+    lines.push(`${arg.name}  ${details.lineCount} lines`)
+    for (const previewLine of details.previewLines) {
+      lines.push(`  ${previewLine}`)
+    }
+    lines.push("")
+  }
+  return lines.join("\r\n") + "\r\n"
+}
+
+function envArgSummary(value) {
+  const details = envArgDetails(value)
+  return {
+    type: "pinokio env arg",
+    lines: details.lineCount,
+    preview: details.previewLines.join("\n"),
+    truncated: details.truncated
+  }
+}
+
+function redactEnvArgs(env) {
+  if (!env || typeof env !== "object") {
+    return env
+  }
+  const redacted = { ...env }
+  for (const key of Object.keys(redacted)) {
+    if (isPinokioEnvArgKey(key)) {
+      redacted[key] = envArgSummary(redacted[key])
+    }
+  }
+  return redacted
+}
+
+module.exports = {
+  renderEnvArgs,
+  quoteEnvArgComposite,
+  hasEnvArgMarker,
+  isPinokioEnvArgKey,
+  formatEnvArgsPreview,
+  redactEnvArgs
+}

package/kernel/index.js

@@ -40,6 +40,7 @@ const Git = require('./git')
 const Connect = require('./connect')
 const Favicon = require('./favicon')
 const AppLauncher = require('./app_launcher')
+const WatchManager = require('./watch')
 const { DownloaderHelper } = require('node-downloader-helper');
 const { ProxyAgent } = require('proxy-agent');
 const fakeUa = require('fake-useragent');
@@ -975,6 +976,7 @@ class Kernel {
     this.loader = new Loader()
     this.bin = new Bin(this)
     this.api = new Api(this)
+    this.watch = new WatchManager(this)
     this.python = new Python(this)
     this.shell = new Shells(this)
     this.appLauncher = new AppLauncher(this)

package/kernel/shell.js

@@ -20,6 +20,7 @@ const { applyWindowsNodePackageManagerEnv } = require('./windows_node_package_ma
 const ShellParser = require('./shell_parser')
 const AnsiStreamTracker = require('./ansi_stream_tracker')
 const ShellStateSync = require('./shell_state_sync')
+const ShellRunTemplate = require('./api/shell_run_template')
 const home = os.homedir()
 
 function normalizeComparablePath(filePath, platform) {
@@ -308,7 +309,7 @@ class Shell {
         delete this.env[key]
       }
       let val = this.env[key]
-      if (/[\r\n]/.test(val)) {
+      if (!ShellRunTemplate.isPinokioEnvArgKey(key) && /[\r\n]/.test(val)) {
         const replaced = val.replaceAll(/[\r\n]+/g, ' ');
         this.env[key] = replaced
 //        delete this.env[key]
@@ -362,6 +363,9 @@ class Shell {
   }
   quoteArgForShell(value, shellName=this.shell) {
     const input = value == null ? "" : String(value)
+    if (ShellRunTemplate.hasEnvArgMarker(input)) {
+      return ShellRunTemplate.quoteEnvArgComposite(input, shellName)
+    }
     if (this.isCmdShell(shellName)) {
       return `"${input.replace(/([()%!^"<>&|])/g, '^$1')}"`
     }
@@ -409,6 +413,7 @@ class Shell {
     this.userActive = false
     this.decsyncBuffer = ''
     this.stateSync.reset()
+    this.envArgsPreviewed = false
 
     /*
       params := {
@@ -540,6 +545,9 @@ class Shell {
         //}
       }
     }
+    if (params._pinokio_cmd_delayed_expansion && this.isCmdShell(this.shell) && !this.args.includes("/V:ON")) {
+      this.args.push("/V:ON")
+    }
 
     // 3. path => path can be http, relative, absolute
     this.path = params.path
@@ -757,6 +765,16 @@ class Shell {
     }
     this.stateSync.invalidate({ clearTail: true })
   }
+  emitEnvArgsPreview(params) {
+    if (!params || !Array.isArray(params._pinokio_env_args) || params._pinokio_env_args.length === 0 || this.envArgsPreviewed) {
+      return
+    }
+    this.envArgsPreviewed = true
+    const raw = ShellRunTemplate.formatEnvArgsPreview(params._pinokio_env_args)
+    if (raw && this.ondata) {
+      this.ondata({ raw })
+    }
+  }
   async run(params, cb) {
     let r = await this.request(params, cb)
     return r
@@ -1362,6 +1380,7 @@ class Shell {
   }
   async exec(params) {
     this.parser = new ShellParser()
+    this.emitEnvArgsPreview(params)
     params = await this.activate(params)
     this.cmd = this.build(params)
     let res = await new Promise((resolve, reject) => {
@@ -1729,7 +1748,7 @@ class Shell {
       cmd: this.cmd,
       index: this.index,
       group: this.group,
-      env: this.env,
+      env: ShellRunTemplate.redactEnvArgs(this.env),
       done: this.done,
       ready: this.ready,
       id: this.id,

package/kernel/watch/context.js

@@ -0,0 +1,42 @@
+const path = require("path")
+const { watchFs } = require("./drivers/fs")
+const { poll } = require("./drivers/poll")
+
+class WatchContext {
+  constructor(options = {}) {
+    this.kernel = options.kernel
+    this.manager = options.manager
+    this.id = options.id
+    this.cwd = path.resolve(options.cwd)
+    this.dirname = path.resolve(options.dirname || options.cwd)
+    this.request = options.request
+    this.script = options.script
+    this.declaration = options.declaration
+    this.input = options.input || {}
+    this.args = options.args || this.input
+    this.watch = {
+      fs: (targetPath, callback, watchOptions = {}) => {
+        return watchFs(this.resolve(targetPath), callback, {
+          ...watchOptions,
+          onError: watchOptions.onError || ((error) => {
+            console.warn("[watch.fs]", error && error.message ? error.message : error)
+          })
+        })
+      }
+    }
+  }
+
+  resolve(targetPath) {
+    return this.kernel.api.resolvePath(this.cwd, String(targetPath || "."))
+  }
+
+  resolveModule(targetPath) {
+    return this.kernel.api.resolvePath(this.dirname, String(targetPath || "."))
+  }
+
+  poll(interval, callback, options = {}) {
+    return poll(interval, callback, options)
+  }
+}
+
+module.exports = WatchContext

package/kernel/watch/drivers/fs.js

@@ -0,0 +1,71 @@
+const fs = require("fs")
+const path = require("path")
+const ParcelWatcher = require("@parcel/watcher")
+
+const DEFAULT_IGNORE = [
+  "**/.git/**",
+  "**/node_modules/**",
+  "**/__pycache__/**",
+  "**/.venv/**",
+  "**/venv/**",
+  "**/env/**"
+]
+
+function isInside(candidate, parent) {
+  const relative = path.relative(parent, candidate)
+  return relative === "" || (!!relative && !relative.startsWith("..") && !path.isAbsolute(relative))
+}
+
+async function nearestExistingDirectory(targetPath) {
+  let current = path.resolve(targetPath)
+  while (current && current !== path.dirname(current)) {
+    const stats = await fs.promises.stat(current).catch(() => null)
+    if (stats && stats.isDirectory()) {
+      return current
+    }
+    current = path.dirname(current)
+  }
+  return current || path.parse(path.resolve(targetPath)).root
+}
+
+async function watchFs(targetPath, callback, options = {}) {
+  const resolvedTarget = path.resolve(targetPath)
+  const targetStats = await fs.promises.stat(resolvedTarget).catch(() => null)
+  const watchRoot = targetStats && targetStats.isDirectory()
+    ? resolvedTarget
+    : await nearestExistingDirectory(resolvedTarget)
+  const filterToTarget = watchRoot !== resolvedTarget
+
+  const subscription = await ParcelWatcher.subscribe(
+    watchRoot,
+    (error, events) => {
+      if (error) {
+        if (typeof options.onError === "function") {
+          options.onError(error)
+        }
+        return
+      }
+      const normalizedEvents = Array.isArray(events) ? events : []
+      const filteredEvents = filterToTarget
+        ? normalizedEvents.filter((event) => event && event.path && isInside(path.resolve(event.path), resolvedTarget))
+        : normalizedEvents
+      if (filteredEvents.length === 0) {
+        return
+      }
+      callback(filteredEvents)
+    },
+    {
+      ignore: Array.isArray(options.ignore) ? options.ignore : DEFAULT_IGNORE
+    }
+  )
+
+  return async () => {
+    if (subscription && typeof subscription.unsubscribe === "function") {
+      await subscription.unsubscribe().catch(() => {})
+    }
+  }
+}
+
+module.exports = {
+  watchFs
+}

package/kernel/watch/drivers/poll.js

@@ -0,0 +1,33 @@
+function poll(interval, callback, options = {}) {
+  const delay = Math.max(100, Number(interval || options.interval || 1000))
+  let stopped = false
+  let running = false
+
+  const tick = async () => {
+    if (stopped || running) return
+    running = true
+    try {
+      await callback()
+    } catch (error) {
+      if (typeof options.onError === "function") {
+        options.onError(error)
+      }
+    } finally {
+      running = false
+    }
+  }
+
+  if (options.immediate !== false) {
+    setTimeout(tick, 0)
+  }
+  const timer = setInterval(tick, delay)
+
+  return async () => {
+    stopped = true
+    clearInterval(timer)
+  }
+}
+
+module.exports = {
+  poll
+}