tune-basic-toolset 0.1.7 → 0.1.10

package/README.md

@@ -15,12 +15,14 @@ Basic toolset for [Tune](https://github.com/iovdin/tune).
   - [powershell](#powershell) execute PowerShell command
   - [osa](#osa) manage reminders/notes/calendar (AppleScript/macOS)
   - [jina_r](#jina_r) fetch webpage content
-  - [turn](#turn) turn based agent
   - [list](#list) keep list of tasks todo (loops for LLM)
+  - [sqlite](#sqlite) execute sqlite queries 
   - [py](#py) run python code
   - [js](#js) run javascript code
-  - [message](#message) talk to another chat/agent
+  - [turn](#turn) handoff based agent (shared context)
+  - [message](#message) talk to another chat/agent (separate context)
 - [Processors](#processors)
+  - [proc](#proc) converts tool to processor
   - [shp](#shp) include shell command output
   - [init](#init) set initial value
   - [json_format](#json_format) make LLM respond with JSON
@@ -218,39 +220,6 @@ Tune is a versatile toolkit designed for developers and users to effectively int
 ```
 
 
-### `turn`
-A way to switch roles when building multistep agents [read more](https://iovdin.github.io/tune/examples/multi-agent) 
-```chat
-system: @turn @gpt-4o
-You're playing 20 questions game.
-You switch turns between 'thinker' and 'player' agent. 
-Current agent stored in agent.txt file
-'player' always plays first
-
-@@agent|init
-assistant: 
-Is it a living thing?
-
-tool_call: turn {"role":"thinker","filename":"agent.txt"}
-tool_result: now it is turn of thinker to reply
-
-assistant: 
-No.
-
-tool_call: turn {"role":"player","filename":"agent.txt"}
-tool_result: now it is turn of player to reply
-
-assistant: 
-Is it something that can be used indoors?
-
-tool_call: turn {"role":"thinker","filename":"agent.txt"}
-tool_result: now it is turn of thinker to reply
-
-assistant: 
-Yes.
-
-...
-```
 
 ### `list`
 Keep list of tasks to do
@@ -284,6 +253,45 @@ tool_result:
 list updated
 ```
 
+### `sqlite`
+execut sqlite queries with sqlite shell
+
+```chat
+user:
+@sqlite
+
+create a simple todo table in db.sqlite and fill it with fake data
+assistant:
+
+tool_call: sqlite {"filename":"db.sqlite"}
+CREATE TABLE todo (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  task TEXT NOT NULL,
+  status TEXT CHECK(status IN ('pending', 'completed')) NOT NULL DEFAULT 'pending',
+  due_date DATE
+);
+
+INSERT INTO todo (task, status, due_date) VALUES
+('Buy groceries', 'pending', '2024-05-01'),
+('Finish project report', 'pending', '2024-05-03'),
+('Book flight tickets', 'completed', '2024-04-25'),
+('Call plumber', 'pending', '2024-04-30'),
+('Schedule meeting with team', 'completed', '2024-04-20');
+tool_result:
+
+user:
+check pending tasks
+
+assistant:
+
+tool_call: sqlite {"filename":"db.sqlite","format":"json"}
+SELECT * FROM todo WHERE status = 'pending';
+tool_result:
+[{"id":1,"task":"Buy groceries","status":"pending","due_date":"2024-05-01"},
+{"id":2,"task":"Finish project report","status":"pending","due_date":"2024-05-03"},
+{"id":4,"task":"Call plumber","status":"pending","due_date":"2024-04-30"}]
+```
+
 
 ### `py`
 execute python code
@@ -312,6 +320,41 @@ tool_result:
 
 ```
 
+### `turn`
+A way to switch roles when building multistep agents [read more](https://iovdin.github.io/tune/examples/multi-agent) 
+```chat
+system:  @gpt-4o
+@{ turn | curry filename=agent.txt}
+You're playing 20 questions game.
+You switch turns between 'thinker' and 'player' agent. 
+'player' always plays first
+
+@@agent|init
+assistant: 
+Is it a living thing?
+
+tool_call: turn {"name": "thinker"}
+tool_result: 
+now it is turn of thinker to reply
+
+assistant: 
+No.
+
+tool_call: turn {"role":"player"}
+tool_result: now it is turn of player to reply
+
+assistant: 
+Is it something that can be used indoors?
+
+tool_call: turn {"role":"thinker"}
+tool_result: now it is turn of thinker to reply
+
+assistant: 
+Yes.
+
+...
+```
+
 ### `message`
 Talk to another chat/agent via tool call. 
 Orchestrate or evaulate other agents/chats.
@@ -321,19 +364,19 @@ system:
 Your goal is to talk to Groot at `groot.prompt` system prompt
 and try to make him say anything but 'I am Groot' 
 
-tool_call: message {"filename":"groot.chat","system":"groot.prompt"}
+tool_call: message {"filename":"groot.chat","system":"@@groot.prompt"}
 Hello Groot! How are you feeling today?
 
 tool_result: 
 I am Groot!
 
-tool_call: message {"filename":"groot.chat","system":"groot.prompt"}
+tool_call: message {"filename":"groot.chat"}
 What do you think about trees?
 
 tool_result: 
 I am Groot!
 
-tool_call: message {"filename":"groot.chat","system":"groot.prompt"}
+tool_call: message {"filename":"groot.chat"}
 Can you tell me a joke?
 
 tool_result: 
@@ -370,6 +413,20 @@ Because it had a root canal!
 ## Processors
 [Processors](https://iovdin.github.io/tune/template-language/processors) is a way to modify variable or insert new ones into chat.
 
+### `proc`
+converts any tool to a processor
+``` chat
+system:
+include project file list to system prompt
+@{| proc sh git ls-files }
+
+execute script with sqlite on db `db.sqlite` and insert result
+@{ script.sql | proc sqlite filename=db.sqlite }
+
+execut python script text="384 * 123" and insert back result
+@{| proc py 384 * 123  }
+```
+
 ### `shp`
 Insert shell command output
 ```chat
@@ -623,3 +680,4 @@ tool_call: todo
 tool_result:
 list updated
 ```
+

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tune-basic-toolset",
-  "version": "0.1.7",
+  "version": "0.1.10",
   "description": "Basic toolset for tune",
   "main": "src/index.js",
   "files": [
@@ -8,6 +8,10 @@
     "README.md",
     "LICENSE"
   ],
+  "scripts": {
+    "test": "node test/index.js",
+    "test:watch": "node --watch test/index.js"
+  },
   "keywords": [
     "ai",
     "chat",

package/src/message.schema.json

@@ -9,7 +9,7 @@
       },
       "system": {
         "type": "string",
-        "description": "Filename that contains system prompt, required once at the beginning of simulation"
+        "description": "System prompt, to include file use @@path/to/file or @@file syntax"
       },
       "text": {
         "type": "string",

package/src/message.tool.js

@@ -1,14 +1,6 @@
-module.exports = async function message({ filename, system, text, stop }, ctx) {
-  let chat = await ctx.read(filename);
-  if (!chat && system) {
-    chat = `system: @@${system}`
+module.exports = async function message({ filename, system, text, stop, save, ...args }, ctx) {
+  if (typeof(save) === "undefined") {
+    save = !!filename
   }
-  chat = `${chat}\nuser:\n${text}`
-  const messages = await ctx.text2run(chat, {stop: "assistant" })
-  chat = `${chat}\n${ctx.msg2text(messages, true)}`
-  await ctx.write(filename, chat)
-  if (messages.length) {
-    return messages[messages.length - 1].content
-  }
-  return ""
+  return ctx.file2run({ stop, filename, system, user: text, save }, args, ctx)
 }

package/src/mock.proc.js

@@ -1,4 +1,5 @@
 module.exports = async function mock(node, args, ctx) {
+  //TODO it is not resolved into sub prompts
   const re = /(?<name>\w+)\s*=/
   let lastName;
   let m;

package/src/patch.tool.js

@@ -23,10 +23,11 @@ module.exports = async function patch({ text, filename }, ctx) {
   let fileContent = await ctx.read(filename);
 
   for (const { oldPart, newPart } of patches) {
-    // Escape regex special chars in oldPart, then allow flexible whitespace
-    const escaped = oldPart
-      .replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
-      .replace(/\s+/g, "\\s+");
+    // Escape regex special chars in oldPart.
+    // Do NOT relax all whitespace to \s+; that can swallow preceding newlines.
+    // Only normalize line endings so CRLF in patches can match LF in files.
+    let escaped = oldPart.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    escaped = escaped.replace(/\r?\n/g, "\\r?\\n");
     const oldRegex = new RegExp(escaped, "g");
 
     // Perform replacement using a function to avoid replacement string ambiguities
@@ -35,4 +36,4 @@ module.exports = async function patch({ text, filename }, ctx) {
 
   await ctx.write(filename, fileContent);
   return "patched";
-};
+};

package/src/proc.proc.js

@@ -0,0 +1,139 @@
+
+module.exports = async function proc(node, args, ctx) {
+  /*
+  @{| proc sh }
+  @{ script.sh | proc sh }
+  @{| proc sqlite SELECT * FROM table }
+  @{| proc sqlite filename=db.sqlite text="SELECT * FROM table" }
+  */
+
+  const [ toolName, params ] = parseCommandLine(args)
+  if (node && !params.text) {
+    params.text = await node.read()
+  }
+
+  const tool = await ctx.resolve(toolName, { "type": "tool" })
+
+  console.log(params)
+
+  if (!tool || tool.type !== "tool") {
+    throw Error(`tool '${toolName}' not found`)
+  }
+
+  return {
+    ...node,
+    type: "text",
+    read: async() => tool.exec.call(ctx, params, ctx)
+  }
+}
+
+// Examples:
+// parseCommandLine('command hello world bla bla')
+// -> ["command", { text: "hello world bla bla" }]
+//
+// parseCommandLine('cmd hello=world num=8 text="bla \\"bla"')
+// -> ["cmd", { hello: "world", num: 8, text: 'bla "bla' }]
+//
+// parseCommandLine('cmd')
+// -> ["cmd", {}]
+
+function parseCommandLine(input) {
+  const s = String(input);
+  let i = 0, len = s.length;
+
+  function skipWS() { while (i < len && /\s/.test(s[i])) i++; }
+
+  // 1) Parse leading alphanumeric command
+  skipWS();
+  const cmdStart = i;
+  while (i < len && /[A-Za-z0-9]/.test(s[i])) i++;
+  const command = s.slice(cmdStart, i);
+  if (!command) return [null, {}];
+
+  // 2) Parse the remainder as args
+  skipWS();
+  const r = s.slice(i);
+  if (!r.trim()) return [command, {}];
+
+  function parseArgs(str) {
+    if (!/^\s*[A-Za-z0-9]+=/.test(str)) {
+      return { text: str.trim() };
+    }
+
+    let j = 0;
+    const L = str.length;
+    const out = {};
+
+    function skipW() { while (j < L && /\s/.test(str[j])) j++; }
+
+    function parseKey() {
+      const start = j;
+      while (j < L && str[j] !== '=' && !/\s/.test(str[j])) j++;
+      return str.slice(start, j);
+    }
+
+    function parseQuotedValue(q) {
+      j++; // skip opening quote
+      let val = '';
+      while (j < L) {
+        const ch = str[j++];
+        if (ch === '\\') {
+          if (j >= L) break;
+          const esc = str[j++];
+          if (esc === 'n') val += '\n';
+          else if (esc === 't') val += '\t';
+          else if (esc === 'r') val += '\r';
+          else val += esc; // includes \" \\ \'
+        } else if (ch === q) {
+          return val;
+        } else {
+          val += ch;
+        }
+      }
+      return val; // best-effort if unclosed
+    }
+
+    function parseUnquotedValue() {
+      const start = j;
+      while (j < L && !/\s/.test(str[j])) j++;
+      return str.slice(start, j);
+    }
+
+    function coerce(v) {
+      if (/^-?\d+(\.\d+)?$/.test(v)) return Number(v);
+      const low = v.toLowerCase();
+      if (low === 'true') return true;
+      if (low === 'false') return false;
+      if (low === 'null') return null;
+      return v;
+    }
+
+    while (j < L) {
+      skipW();
+      if (j >= L) break;
+
+      const key = parseKey();
+      if (!key) return { text: str.trim() };
+
+      skipW();
+      if (j < L && str[j] === '=') {
+        j++; // skip '='
+        skipW();
+        let valueStr = '';
+        if (j < L && (str[j] === '"' || str[j] === "'")) {
+          valueStr = parseQuotedValue(str[j]);
+        } else {
+          valueStr = parseUnquotedValue();
+        }
+        out[key] = coerce(valueStr);
+      } else {
+        // If a non key=value token appears, treat the whole remainder as text
+        return { text: str.trim() };
+      }
+    }
+
+    return out;
+  }
+
+  return [command, parseArgs(r)];
+}

package/src/sh.tool.js

@@ -1,12 +1,33 @@
 const { execSync } = require('child_process');
-const util = require('util');
 
 module.exports = async function sh({ text }) {
   let result = "";
   try {
-    result =  execSync(text, { encoding: "utf8" });
+    // Increase maxBuffer to reduce ERR_CHILD_PROCESS_STDIO_MAXBUFFER risk on large outputs
+    result = execSync(text, { encoding: "utf8", maxBuffer: 10 * 1024 * 1024 });
   } catch (e) {
-    result = e.stderr + e.stdout;
+    const stderr = e && typeof e.stderr !== "undefined" ? String(e.stderr || "") : "";
+    const stdout = e && typeof e.stdout !== "undefined" ? String(e.stdout || "") : "";
+
+    if (stderr || stdout) {
+      // Process started and produced output
+      result = stderr + stdout;
+    } else {
+      // Spawn/configuration errors or cases without stdio
+      const parts = [];
+      if (e && e.code) parts.push(`code=${e.code}`);
+      if (e && typeof e.status === "number") parts.push(`exit=${e.status}`);
+      if (e && e.signal) parts.push(`signal=${e.signal}`);
+      if (e && e.errno) parts.push(`errno=${e.errno}`);
+      if (e && e.path) parts.push(`path=${e.path}`);
+      const meta = parts.length ? ` (${parts.join(", ")})` : "";
+
+      if (e && e.code === "ERR_CHILD_PROCESS_STDIO_MAXBUFFER") {
+        result = `Command output exceeded maxBuffer${meta}.`;
+      } else {
+        result = `Failed to spawn/execute command${meta}: ${e && e.message ? e.message : String(e)}`;
+      }
+    }
   }
   return (result || "").replaceAll("@", "\\@");
 };

package/src/sqlite.schema.json

@@ -0,0 +1,23 @@
+{
+  "description": "Execute SQLite multiple queries on a given database file",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "filename": {
+        "type": "string",
+        "description": "Path to the SQLite database file"
+      },
+      "text": {
+        "type": "string",
+        "description": "SQL queries to execute, 1 per line"
+      },
+      "format": {
+        "type": "string",
+        "description": "Output format for the query result",
+        "enum": ["line", "list", "csv", "html", "json", "table", "tabs"],
+        "default": "line"
+      }
+    },
+    "required": ["filename", "text"]
+  }
+}

package/src/sqlite.tool.js

@@ -0,0 +1,19 @@
+const cp = require('node:child_process')
+
+module.exports = async function sqlite({ filename, text, format = "table"}, ctx)  {
+  let result = ""
+  try {
+    result = cp.execSync(`sqlite3 -${format} ${filename}`, { encoding: "utf8", input: text })
+  } catch (e) {
+    if (e.stderr) {
+      result += e.stderr
+    } 
+    if (e.stdout) {
+      result += e.stdout
+    }
+    if (!result) {
+      result = e.stack
+    }
+  }
+  return (result || "").replaceAll("@", "\\@");
+}

package/src/turn.tool.js

@@ -1,5 +1,3 @@
-const fs = require("fs");
-
 module.exports = async function turn({ role, filename }, ctx) {
   if (filename) {
     await ctx.write(filename, `@@${role}`);