tune-basic-toolset 0.1.15 → 0.1.17

package/README.md

@@ -10,7 +10,7 @@ Basic toolset for [Tune](https://github.com/iovdin/tune).
   - [wf](#wf) write file
   - [patch](#patch) patch file  
   - [append](#append) append to file
-  - [sh](#sh) execute shell command
+  - [sh](#sh) execute shell command locally and remotely
   - [cmd](#cmd) execute Windows cmd command
   - [powershell](#powershell) execute PowerShell command
   - [grep](#grep) search for patterns in text or files
@@ -20,12 +20,12 @@ Basic toolset for [Tune](https://github.com/iovdin/tune).
   - [list](#list) keep list of tasks todo (loops for LLM)
   - [sqlite](#sqlite) execute sqlite queries 
   - [py](#py) run python code
-  - [js](#js) run javascript code
+  - [js_node](#js_node) run javascript code in node process
+  - [js_ctx](#js_ctx) run javascript code that shares context with LLM
   - [turn](#turn) handoff based agent (shared context)
   - [message](#message) talk to another chat/agent (separate context)
 - [Processors](#processors)
   - [proc](#proc) converts tool to processor
-  - [shp](#shp) include shell command output
   - [init](#init) set initial value
   - [json_format](#json_format) make LLM respond with JSON
   - [log](#log) save LLM payload
@@ -130,7 +130,7 @@ appended
 ```
 
 ### `sh`
-Execute shell command
+Execute shell command locally or with ssh
 ```chat
 user: @sh
 find with ripgrep where echo is used
@@ -141,6 +141,15 @@ tool_result:
 ./README.md:  const text = "s: \@echo\nu: hello world";
 ./tools/echo.txt:you are echo, you print everything back
 ./tools/README.md:* `echo.txt` - to debug variable expansions and context
+
+user:
+check contents directory on root\@host.com
+assistant:
+tool_call: sh {"host": "root@host.com"}
+ls
+tool_result:
+file1
+file2
 ```
 
 ### `cmd`
@@ -342,10 +351,10 @@ tool_result:
 5545
 ```
 
-### `js`
-execute javascript code
+### `js_node`
+execute javascript code in nodejs
 ```chat
-user: @js
+user: @js_node
 1234 + 4311
 
 tool_call: js {"inputType":"commonjs"}
@@ -356,6 +365,85 @@ tool_result:
 
 ```
 
+### `js_ctx`
+execute javascript code but also call tools available to llm or trigger LLM response from code 
+It is useful when you need to trigger loops and ifs or chain few tool calls together
+```chat
+user: 
+@js_ctx @sh
+1234 + 4311
+
+tool_call: sh
+ls
+tool_result:
+__pycache__
+append.schema.json
+append.tool.js
+... file list
+
+user:
+can you do the same using js_ctx tool?
+
+assistant:
+
+tool_call: js_ctx
+
+const result = await ctx.exec("sh", { text: "ls" });
+return result;
+
+tool_result:
+__pycache__
+append.schema.json
+append.tool.js
+....
+
+user:
+can you loop throught the .js files and summarize its content using llm?
+assistant:
+
+tool_call: js_ctx
+
+// Get all .js files (excluding .proc.js and .schema.json files)
+const files = await ctx.exec("sh", { text: "ls *.js | grep -v '.proc.js'" });
+const jsFiles = files.trim().split('\n').filter(f => f.endsWith('.js'));
+
+const summaries = [];
+
+// Loop through each .js file
+for (const file of jsFiles) {
+  // Read the file content
+  const content = await ctx.exec("sh", { text: `cat ${file}` });
+  
+  // Use LLM to summarize
+  const summary = await ctx.file2run({
+    system: "@mistral-tiny You are a code analyst. Provide a brief 1-2 sentence summary of what this JavaScript file does.",
+    user: "Summarize this code:\n\n@code"
+  }, {
+    code: content
+  });
+  
+  summaries.push({
+    file: file,
+    summary: summary.trim()
+  });
+}
+
+// Format and return results
+return summaries.map(s => `**${s.file}**\n${s.summary}\n`).join('\n');
+
+tool_result:
+**append.tool.js**
+This JavaScript file exports an asynchronous function that appends the given text to the end of a specified file, and returns "done" once the operation is completed.
+
+**cmd.tool.js**
+This JavaScript file exports an asynchronous function `cmd` that executes a system command (Windows-specific, using 'cmd.exe' as the shell) passed as a string, and returns the command's output or an error message containing both stderr and stdout.
+
+**grep.tool.js**
+This JavaScript module exports an asynchronous function named `grep` that reads a file (if provided) or a provided text and searches for lines matching a given regular expression. It returns the lines that match, separated by newlines. If the file is not found or the content is empty, it returns appropriate error messages.
+....
+
+```
+
 ### `turn`
 A way to switch roles when building multistep agents [read more](https://iovdin.github.io/tune/examples/multi-agent) 
 ```chat
@@ -463,29 +551,6 @@ execut python script text="384 * 123" and insert back result
 @{| proc py 384 * 123  }
 ```
 
-### `shp`
-Insert shell command output
-```chat
-system:
-include project file list to system prompt
-@{| shp git ls-files }
-
-include buffer content on osx
-@{| shp pbpaste }
-
-include current date
-@{| shp date }
-
-pipe filename content to shell command
-@{ a.log | shp tail }
-
-@{ a.log | shp grep pattern }
-
-print screen of one of tmux session
-@{| shp tmux capture-pane -t 0 -p }
-
-```
-
 ### `init` 
 Set default value for non set variables
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tune-basic-toolset",
-  "version": "0.1.15",
+  "version": "0.1.17",
   "description": "Basic toolset for tune",
   "main": "src/index.js",
   "files": [

package/src/js_ctx.schema.json

@@ -0,0 +1,13 @@
+{
+  "description": "run js code",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "text": {
+        "type": "string",
+        "description": "js code to execute. You can use 'ctx' inside code to call tools available, e.g. await ctx.exec(\"tool_name\", { param: value }),  it is also possible to call LLM from code:   await ctx.file2run({ system: \"You are helpful assistant\" user: \"user request: @data \"  }, { data: \"value of template variable\") to proper escape user input file2run template engine instead of javascripts ${}"
+      }
+    },
+    "required": ["text"]
+  }
+}

package/src/js_ctx.tool.js

@@ -0,0 +1,48 @@
+const { parseScript } = require('esprima');
+const { generate } = require('escodegen');
+const vm = require('node:vm');
+module.exports = async function js({ text }, ctx) {
+
+  let code = `(async () =>{\n${text}\n})()`
+  const ast = parseScript(code, { range: true });
+
+  const bodyAst = ast.body[ast.body.length - 1].expression.callee.body;
+  const lastNode = bodyAst.body[bodyAst.body.length - 1];
+
+  if (lastNode.type === 'ExpressionStatement') {
+    bodyAst.body[bodyAst.body.length - 1] = {
+      type: 'ReturnStatement',
+      argument: lastNode.expression
+    }
+    code = generate(ast);
+  }
+  if (bodyAst.length === 0) {
+    return null;
+  }
+
+  let logs = [];
+
+  try {
+    let result = vm.runInContext(code, vm.createContext({ 
+      ctx,
+      console: {
+        log: (...args) => logs.push(args),
+        error: (...args) => logs.push(args),
+        warn: (...args) => logs.push(args),
+        debug: (...args) => logs.push(args),
+      }
+    }))
+    result = await result
+    return result
+    if (typeof(result) === "object") {
+      result = JSON.stringify(result, null, "  ")
+    } else {
+      result = String(result)
+    }
+    logs = logs.map(log => log.join(" ")).join("\n")
+    result = logs.trim() + "\n" +  (result || "").trim()
+    return result.trim()
+  } catch (e) {
+    return e.stack
+  }
+}

package/src/{js.schema.json → js_node.schema.json}

@@ -1,5 +1,5 @@
  {
-  "description": "Execute a given JavaScript code snippet and return the result",
+  "description": "Execute a given JavaScript code snippet and return the result using new node process",
   "parameters": {
     "type": "object",
     "properties": {

package/src/sh.schema.json

@@ -6,6 +6,10 @@
       "text": {
         "type": "string",
         "description": "The shell command to execute"
+      },
+      "host": {
+        "type": "string",
+        "description": "remote host like user@host.com to execute the shell comand on (uses ssh)"
       }
     },
     "required": ["text"]

package/src/sh.tool.js

@@ -1,11 +1,23 @@
-const { execSync } = require('child_process');
+const { spawnSync, execSync } = require('child_process');
 
-module.exports = async function sh({ text }) {
+//TODO look back do not escape escaped
+const escape = (str) => str.replace(/'/g, "\\'")
+
+module.exports = async function sh({ text, host }) {
   let result = "";
   try {
+    let cmd = text
+    let args = []
+    if (host) {
+      cmd = "ssh"
+      args = [host, text]
+    }
     // Increase maxBuffer to reduce ERR_CHILD_PROCESS_STDIO_MAXBUFFER risk on large outputs
-    result = execSync(text, { encoding: "utf8", maxBuffer: 10 * 1024 * 1024 });
+    result = spawnSync(cmd, args, { encoding: "utf8", maxBuffer: 10 * 1024 * 1024, shell: true });
+    result = (result.stdout || "") + (result.stderr || "")
+    return result
   } catch (e) {
+    return e
     const stderr = e && typeof e.stderr !== "undefined" ? String(e.stderr || "") : "";
     const stdout = e && typeof e.stdout !== "undefined" ? String(e.stdout || "") : "";
 

package/src/shp.proc.js

@@ -1,31 +0,0 @@
-const { execSync, spawnSync } = require('child_process');
-
-const shp = async (node, args, ctx) => ({
-  type: 'text',
-  read: async () => {
-    let input = null;
-
-    if (node && node.type === 'text') {
-      input = await node.read();
-    }
-
-    let result;
-    try {
-      if (input !== null) {
-        const res = spawnSync(args.trim() || "sh", {
-          input,
-          encoding: 'utf8',
-          shell: true
-        });
-        result = (res.stdout || '') + (res.stderr || '');
-      } else {
-        result = execSync(args.trim(), { encoding: 'utf8' });
-      }
-    } catch (e) {
-      result = e.stderr + e.stdout;
-    }
-    return result;
-  }
-});
-
-module.exports = shp;