tune-sdk 0.2.24 → 0.3.1

package/README.md

@@ -6,8 +6,7 @@ Tune is a handy [extension for Visual Studio Code and](https://marketplace.visua
 With tune [javascript sdk](https://www.npmjs.com/package/tune-sdk) you can make apps and agents. 
 
 ## Demo
-<video src="https://github.com/user-attachments/assets/23f8ab30-58db-4159-8761-f212a7960e0c">
-</video>
+[![asciicast](https://asciinema.org/a/757894.png)](https://asciinema.org/a/757894)
 
 
 ## Setup
@@ -15,6 +14,7 @@ install tune-sdk
 ```bash
 npm install -g tune-sdk
 
+# create ~/.tune folder and install batteries
 tune init
 ```
 
@@ -40,52 +40,15 @@ user:
 ```
 [read more](https://iovdin.github.io/tune/template-language)
 
-
-## Diagram
-
-```mermaid
-flowchart TD 
-
-  subgraph Entry[" "]
-    Editor["VSCode/Neovim/Sublime Text"]
-    CLI["CLI"]
-    App["App"]
-  end
-
-  subgraph Core[" "]
-    MD1["~/.tune/default.ctx.js"]
-    MD2["require('tune-fs')
-require('tune-models')
-"]
-    CTX["tune.makeContext(...middlewares)"]
-    F2R["ctx.file2run(params)"]
-  end
-
-  
-  MD1 --> |cli middlewares| CTX
-  MD2 --> |app middlewares| CTX
-  Editor -->| $ tune rpc | Core
-  CLI --> | $ tune --user hello | Core
-  App --> Core
-  
-  
-  
-  F2R -->|ctx.resolve #40; system.txt #124; shell #124; gpt-5  #41; | CTX
-  CTX -->| #123; type: text #124; tool #124; llm #125; | F2R
-
-  F2R --> |fetch| LLM["https://provider.com/v1/chat/completions"]
-  F2R --> |call| Tool
-```
-
 ## Extend with Middlewares
 Extend Tune with middlewares:
 
-* [tune-fs](https://www.npmjs.com/package/tune-fs) - connect tools & files from local filesystem
-* [tune-models](https://www.npmjs.com/package/tune-models) - connect llm models from Anthropic/OpenAI/Gemini/Openrouter/Mistral/Groq
-* [tune-basic-toolset](https://www.npmjs.com/package/tune-basic-toolset) - basic tools like read file, write file, shell etc.
-* [tune-s3](https://www.npmjs.com/package/tune-s3) - read/write files from s3
-* [tune-mcp](https://www.npmjs.com/package/tune-mcp) - connect tools from mcp servers
-* [maik](https://www.npmjs.com/package/@iovdin/maik) - fetch all you emails, and index them into sqlite database
+* [tune-fs](https://github.com/iovdin/tune-fs) - connect tools & files from local filesystem
+* [tune-models](https://github.com/iovdin/tune-models) - connect llm models from Anthropic/OpenAI/Gemini/Openrouter/Mistral/Groq
+* [tune-basic-toolset](https://github.com/iovdin/tune-basic-toolset) - basic tools like read file, write file, shell etc.
+* [tune-s3](https://github.com/iovdin/tune-s3) - read/write files from s3
+* [tune-mcp](https://github.com/iovdin/tune-mcp) - connect tools from mcp servers
+* [maik](https://github.com/iovdin/maik) - fetch all you emails, and index them into sqlite database
 
 
 For example:
@@ -137,6 +100,8 @@ image generated
 # install tune globally
 npm install -g tune-sdk
 
+tune "hi how are you?"
+
 # append user message to newchat.chat run and save
 tune --user "hi how are you?" --filename newchat.chat  --save
 
@@ -144,35 +109,114 @@ tune --user "hi how are you?" --filename newchat.chat  --save
 # print result to console
 tune --system "You are Groot" --user "Hi how are you?"
 
-#set context variable
-tune --set-test "hello" --user "@test" --system "You are echo you print everythting back"  
+# set context variable
+tune --set test="hello" --user "@test" --system "You are echo you print everythting back"  
+# prints hello
+
 ```
 
 
 ## Javascript SDK
 `npm install tune-sdk`
 
-```javascript
-const tune = require("tune-sdk");
-const sonnet = require("./sonnet.llm.js");
+Tune core is middleware-based. A context resolves `@name` references into nodes like `text`, `tool`, `llm`, and `processor`.
 
-require('dotenv').config();
+```javascript
+const tune = require("tune-sdk")
 
 async function main() {
-  const ctx = tune.makeContext({
-    echo: "You are echo, you print everything back",
-    OPENROUTER_KEY: process.env.OPENROUTER_KEY,
-    "default": {
-      type: "llm",
-      exec: sonnet
+  const ctx = tune.makeContext()
+
+  ctx.use(async function middleware(name) {
+    if (name === "file.txt") {
+      return {
+        type: "text",
+        name: "file.txt",
+        read: async () => fs.readFileSync("file.txt", "utf8")
+      }
+    }
+
+    if (name === "readfile") {
+      return {
+        type: "tool",
+        name: "readfile",
+        schema: {
+          type: "object",
+          properties: {
+            filename: { type: "string" }
+          }
+        },
+        exec: async ({ filename }) => fs.readFileSync(filename, "utf8")
+      }
+    }
+
+    if (name === "gpt-5") {
+      return {
+        type: "llm",
+        name: "gpt-5",
+        exec: async (payload) => ({
+          url: "https://api.openai.com/v1/chat/completions",
+          method: "POST",
+          headers: {
+            Authorization: `Bearer ${process.env.OPENAI_KEY}`,
+            "Content-Type": "application/json"
+          },
+          body: JSON.stringify({
+            model: "gpt-5",
+            ...payload
+          })
+        })
+      }
+    }
+
+    if (name === "tail") {
+      return {
+        type: "processor",
+        name: "tail",
+        exec: async (node, args) => {
+          if (!node) return
+          if (node.type !== "text") throw Error("tail can only modify text nodes")
+          return {
+            ...node,
+            read: async () => {
+              const content = await node.read()
+              const n = parseInt(args.trim(), 10) || 20
+              return content.split("\n").slice(-n).join("\n")
+            }
+          }
+        }
+      }
     }
   })
 
-  const text = "s: @echo\nu: hello world";
-  const messages = await tune.text2run(text, ctx)
-  console.log(tune.msg2text(messages))
-  // a: hello world
+  const content = await ctx.file2run({
+    system: "@gpt-5 @readfile",
+    user: "can you read file.txt?",
+    stream: false,
+    response: "content"
+  })
+
+  console.log(content)
 }
+
 main()
 ```
+
 [read more](https://iovdin.github.io/tune/api) about javascript sdk
+
+## Help / Manual
+
+You can access tune manuals and available middlewares manuals from 
+
+```chat
+system:
+@man include all manuals for all connected packages
+@man/ - list all the manuals, like list directory
+@man/tune - get manual for tune core package 
+@man/tune-basic-toolset - get manual for tune-basic-tool set package
+
+tool_call: rf { "filename": "man/tune-basic-toolset"}
+tool_result:
+@man/tune-basic-toolset
+
+```

package/config/default.ctx.js

@@ -1,5 +1,6 @@
 const path = require('path')
 
+const man = require("tune-sdk/man");
 const models = require("tune-models")
 const basics = require("tune-basic-toolset")
 const tunefs = require("tune-fs")
@@ -11,6 +12,7 @@ if (process.env.TUNE_PATH) {
 }
 
 module.exports = [
+  man(),
   current(),
   basics(),
   tunefs({ paths: dirs, makeSchema: true }),

package/package.json

@@ -1,12 +1,12 @@
 {
   "name": "tune-sdk",
-  "version": "0.2.24",
-  "description": "tune - LLM chat in text file",
+  "version": "0.3.1",
+  "description": "chat with llm in a text file. core package",
   "main": "dist/tune.js",
   "module": "dist/tune.mjs",
-  "bin": { 
+  "bin": {
     "tune": "bin/cli.js",
-    "tune-sdk": "bin/cli.js" 
+    "tune-sdk": "bin/cli.js"
   },
   "exports": {
     ".": {
@@ -18,6 +18,12 @@
     },
     "./rpc": {
       "require": "./src/rpc.js"
+    },
+    "./contextws": {
+      "require": "./src/contextws.js"
+    },
+    "./man": {
+      "require": "./src/man.js"
     }
   },
   "keywords": [
@@ -34,5 +40,10 @@
   "scripts": {
     "test": "node test/index.js",
     "test:watch": "node --watch test/index.js"
+  },
+  "dependencies": {
+    "commander": "^14.0.3",
+    "mime-types": "^3.0.2",
+    "ws": "^8.20.0"
   }
 }

package/src/cli.js

@@ -1,6 +1,12 @@
-var assert, tune, rpc, path, fs, os, cp, stream;
-assert = require("assert");
-
+const { Command, Option } = require("commander");
+const tune = require("../dist/tune.js");
+const ws = require("./ws.js");
+const rpc = require("./rpc.js");
+const path = require("path");
+const fs = require("fs");
+const os = require("os");
+const cp = require("child_process");
+const stream = require("stream");
 
 // tune app - run web server from current directory serving index.html and making it availble to call ctx via websocket
 // tune ps - list of executing agents or the ones finished
@@ -10,195 +16,15 @@ assert = require("assert");
 // tune  - execute call/file and quit
 // tune rpc - run rpc server
 
-
-function tpl(str) {
-  var _i;
-  var params = 2 <= arguments.length ? [].slice.call(arguments, 1, _i = arguments.length - 0) : (_i = 1, []);
-  return (function(paramIndex, params) {
-    var _ref;
-    try {
-      _ref = str.replace(/{(\W*)(\w*)(\W*)}/gm, (function(_, pre, name, post) {
-        return (function(res) {
-          paramIndex += 1;
-          return ((typeof res !== 'undefined') ? ((pre || "") + res + (post || "")) : "");
-        })(params[name || paramIndex]);
-      }));
-    } catch (e) {
-      _ref = console.log.apply(console, [].concat([e, str]).concat(params));
-    }
-    return _ref;
-  })(0, (((typeof params[0] === "object") && (params.length === 1)) ? params[0] : params));
-}
-
-function showHelp() {
-  console.log("TUNE-CLI - Command Line Interface for Tune");
-  console.log("");
-  console.log("USAGE:");
-  console.log("  tune [cmd] [OPTIONS]");
-  console.log("");
-  console.log("COMMANDS:");
-  console.log("  rpc                   Start RPC server mode");
-  console.log("  init                  Initialize Tune config directory");
-  console.log("");
-  console.log("EXAMPLES:");
-  console.log("  # Quick chat with system prompt");
-  console.log("  tune --system \"You are Groot\" --user \"Hi how are you?\"");
-  console.log("");
-  console.log("  # Continue existing chat");
-  console.log("  tune --user \"continue the conversation\" --filename chat.chat --save");
-  console.log("");
-  console.log("  # Set context variables");
-  console.log("  tune --set-test=hello --user \"@test\" --system \"Echo assistant\"");
-  console.log("");
-  console.log("  # RPC mode for editor integration");
-  console.log("  tune rpc");
-  console.log("");
-  console.log("  # Initialize or reinitialize config directory");
-  console.log("  tune init --force");
-  console.log("");
-  console.log("OPTIONS:");
-  console.log("  --user <text>         User message to send");
-  console.log("  --system <text>       System prompt to use");
-  console.log("  --filename <file>     Chat file to load/save");
-  console.log("  --save                Save conversation to file");
-  console.log("  --stop <mode>         Stop condition: assistant|step|<custom>");
-  console.log("  --text <content>      chat content");
-  console.log("  --response <type>     Response format: content|json|messages|chat (default: content)");
-  console.log("  --set-<name>=<value>  Set context parameter");
-  console.log("  --path <paths>        Additional search paths (colon-separated)");
-  console.log("  --home <dir>          Tune config directory (default: ~/.tune)");
-  console.log("  --debug               Enable debug output");
-  console.log("  --silent              Suppress output");
-  console.log("  --force               Force config initialization (with 'init')");
-  console.log("  --help                Show this help");
-  console.log("  --version             Show CLI version");
-  return console.log("");
-}
-showHelp;
-
-function validateArgs(args) {
-  assert(!!args && (typeof args === "object"), "Arguments must be an object");
-  if (args.user) assert(typeof args.user === "string", "--user must be a string");
-  if (args.system) assert(typeof args.system === "string", "--system must be a string");
-  if (args.filename) assert(typeof args.filename === "string", "--filename must be a string");
-  if (args.text) assert(typeof args.text === "string", "--text must be a string");
-  if (args.response) assert(typeof args.response === "string", "--response must be a string");
-  if (args.stop) assert(typeof args.stop === "string", "--stop must be a string");
-  if (args.path) assert(typeof args.path === "string", "--path must be a string");
-  if (args.home) assert(typeof args.home === "string", "--home must be a string");
-  if (!!args.save) assert(typeof args.save === "boolean", "--save must be a boolean");
-  if (!!args.debug) assert(typeof args.debug === "boolean" || typeof args.debug === "string", "--debug must be a boolean");
-  if (!!args.silent) assert(typeof args.silent === "boolean", "--silent must be a boolean");
-  if (!!args.force) assert(typeof args.force === "boolean", "--force must be a boolean");
-  if (typeof args.rpc !== "undefined") assert(false, "Use 'tune rpc' instead of --rpc");
-  if (typeof args.forceInit !== "undefined") assert(false, "Use 'tune init --force' instead of --force-init");
-  if (args.params) assert(!!args.params && (typeof args.params === "object"), "--set-* parameters must form a valid object");
-  if ((args.stop && (typeof args.stop === "string"))) assert((args.stop === "assistant") || (args.stop === "step") || (args.stop.length > 0), "--stop must be 'assistant', 'step', or a non-empty custom string");
-  if (args.cmd) {
-    assert(typeof args.cmd === "string", "Command must be a string");
-    assert((args.cmd === "rpc") || (args.cmd === "init"), "Unknown command: " + args.cmd);
-  }
-  if ((!args.help && !args.version && !args.cmd && !args.user && !args.filename && !args.text)) assert(false, "Must specify --user, --filename, a command (rpc|init), --version, or --help");
-  return args;
-}
-validateArgs;
-
-function parseArgs(args) {
-  var curKey, res, res1, key, value, stop, _ref, _len;
-  assert(Array.isArray(args), "parseArgs expects an array of arguments");
-  var curKey;
-  curKey = null;
-  var res;
-  res = args.reduce((function(memo, arg) {
-    var key, value, _ref, _i;
-    assert(typeof arg === "string", "Each argument must be a string");
-    if (arg.startsWith("--")) {
-      _ref = arg.substring(2)
-        .split("=");
-      key = _ref[0];
-      value = _ref[1];
-      assert((typeof key === "string") && (key.length > 0), "Argument key must be a non-empty string");
-      if (!!value) {
-        memo[key] = value;
-        curKey = null;
-      } else {
-        curKey = key;
-        memo[key] = true;
-      }
-    } else if (curKey) {
-      memo[curKey] = arg;
-      curKey = null;
-    } else {
-      if (!memo.__cmd) {
-        memo.__cmd = arg;
-      } else {
-        assert(false, "Only a single positional command is allowed");
-      }
-    }
-    return memo;
-  }), {});
-  assert(!!res && (typeof res === "object"), "Parsed arguments must form an object");
-  var res1;
-  res1 = {};
-  _ref = res;
-  for (key in _ref) {
-    value = _ref[key];
-    assert(typeof key === "string", "Argument keys must be strings");
-    if (key === "__cmd") {
-      res1.cmd = value;
-      continue;
-    }
-    if (key.startsWith("set-")) {
-      res1.params = res1.params || {}
-      assert(key.substr(4).length > 0, "Set parameter name cannot be empty");
-      res1.params[key.substr(4)] = value;
-    } else {
-      res1[key] = value;
-    }
-  }
-  if ((res1.h || res1.help)) res1.help = true;
-  if ((res1.v || res1.version)) res1.version = true;
-  stop = res1.stop;
-  if ((!!stop && (stop !== "step" && stop !== "assistant"))) {
-    assert(typeof stop === "string", "Custom stop condition must be a string");
-    assert(stop.length > 0, "Custom stop condition cannot be empty");
-    res1.stop = (function(msgs) {
-      var lastMsg;
-      assert(Array.isArray(msgs), "Messages must be an array");
-      if (!msgs.length) return false;
-      var lastMsg;
-      lastMsg = msgs["slice"](-1)[0];
-      assert(!!lastMsg && (typeof lastMsg === "object"), "Last message must be an object");
-      if (!lastMsg.content) return false;
-      assert(typeof lastMsg.content === "string", "Message content must be a string");
-      return (-1 !== lastMsg.content.indexOf(stop));
-    });
-  }
-  return res1;
-}
-parseArgs;
-tune = require("../dist/tune.js");
-rpc = require("../src/rpc.js");
-path = require("path");
-fs = require("fs");
-os = require("os");
-cp = require("child_process");
-stream = require("stream");
-
-function getHomedir(args) {
-  assert(!!args && (typeof args === "object"), "getHomedir expects args to be an object");
-  if (args.home) assert(typeof args.home === "string", "args.home must be a string");
-  return path.resolve(path.normalize((args.home || process.env.TUNE_HOME || "~/.tune")
+function getHomedir(home) {
+  return path.resolve(path.normalize((home || process.env.TUNE_HOME || "~/.tune")
     .replace("~", os.homedir())));
 }
-getHomedir;
-async function initConfig(args) {
-  var homedir, stdout, stderr, _ref, _i;
-  assert(!!args && (typeof args === "object"), "initConfig expects args to be an object");
-  var homedir;
-  homedir = getHomedir(args);
-  assert(typeof homedir === "string", "Home directory must be a string");
-  if ((!args.force && fs.existsSync(homedir))) return;
+
+async function initConfig({ home, force }) {
+  let stdout, stderr, _i;
+  const homedir = getHomedir(home);
+  if (!force && fs.existsSync(homedir)) return;
   console.error("[tune] initialize " + homedir);
   fs.mkdirSync(homedir, {
     recursive: true
@@ -207,11 +33,10 @@ async function initConfig(args) {
   fs.cpSync(path.resolve(__dirname, "../config"), path.resolve(homedir), { recursive: true });
   console.error("[tune] installing npm");
   try {
-    _ref = cp.execSync("npm i", {
+    stdout = cp.execSync("npm i", {
       cwd: homedir,
       encoding: "utf8"
     });
-    stdout = _ref;
     if (stdout.trim()) console.error("[tune]", stdout.trim());
     //stderr.trim() ? console.error("[tune]", stderr.trim()) : undefined;
   } catch (err) {
@@ -220,7 +45,7 @@ async function initConfig(args) {
   console.error("[tune] done");
   console.error(`[tune] edit ${homedir}/.env and add OPENAI_KEY and other keys, change ${homedir}/default.ctx.js to customize tune`);
 }
-initConfig;
+
 async function suggest(params, ctx) {
   var node, _ref;
   var node;
@@ -247,7 +72,7 @@ async function suggest(params, ctx) {
     }
   }));
 }
-suggest;
+
 async function remoteContext(name, params) {
   var server, node;
   var server;
@@ -275,20 +100,20 @@ async function remoteContext(name, params) {
   });
   return node;
 }
-remoteContext;
-async function runRpc(args) {
+
+async function runRpc({ debug, home, path }) {
   var inStream, outStream, debugStream, ctx, server;
   inStream = stream.Readable.toWeb(process.stdin);
   outStream = stream.Writable.toWeb(process.stdout);
-  debugStream = ((typeof args.debug === "string") ? fs.createWriteStream(args.debug, {
+  debugStream = ((typeof debug === "string") ? fs.createWriteStream(debug, {
     flags: "a"
   }) : undefined);
-  ctx = await initContext(args);
+  ctx = await initContext({ home, path });
   let cleanCtx = ctx.clone()
   server = rpc.jsonrpc({
     inStream: inStream,
     outStream: outStream,
-    debug: ((typeof args.debug === "string") ? (function() {
+    debug: ((typeof debug === "string") ? (function() {
       var _i;
       var args = 1 <= arguments.length ? [].slice.call(arguments, 0, _i = arguments.length - 0) : (_i = 0, []);
       return debugStream.write(args.join(" ") + "\n", "utf8");
@@ -346,20 +171,13 @@ async function runRpc(args) {
   // console.log("node", node)
   return null;
 }
-runRpc;
-async function run(args) {
-  var ctx, stop, params, res;
-  ctx = await initContext(args);
-  var stop;
-  stop = args.stop || "assistant";
-  var params;
-  params = args.params || {}
-  delete args.params;
-  var res;
-  res = await ctx.file2run({ ...args, errors: "message", stop }, params);
-  return (!args.silent ? console.log(res) : undefined);
+async function run({ home, stop, params, silent, user, system, save, text , response, path, filename }) {
+  const ctx = await initContext({ home, path });
+  const res = await ctx.file2run({ user, system, save, text, response, errors: "message", stop, filename }, params);
+  if (!silent) {
+    console.log(res)
+  }
 }
-run;
 
 function flatten(array) {
   return array.reduce((memo, item) => {
@@ -371,17 +189,17 @@ function flatten(array) {
   }, [])
 }
 
-async function initContext(args) {
+async function initContext({ home, path: addPaths }) {
   var dirs, pwd, ctx, dir, ctxName, ext, module, m, _i, _ref, _len, _i0, _ref0, _len0;
   var dirs;
   var pwd;
   dirs = [];
   pwd = process.cwd();
-  if (args.path) dirs = args.path.split(path.delimiter)
+  if (addPaths) dirs = addPaths.split(path.delimiter)
     .map((function(dir) {
       return path.resolve(pwd, dir);
     }));
-  dirs.push(getHomedir(args));
+  dirs.push(getHomedir(home));
   dirs.unshift(pwd);
   if (process.env.TUNE_PATH) {
     dirs = dirs.concat(process.env.TUNE_PATH.split(path.delimiter))
@@ -389,7 +207,7 @@ async function initContext(args) {
   process.env.TUNE_PATH = dirs.join(path.delimiter);
   ctx = tune.makeContext({
     TUNE_PATH: process.env.TUNE_PATH,
-    TUNE_HOME: getHomedir(args)
+    TUNE_HOME: getHomedir(home)
   });
   _ref = dirs;
   for (_i = 0, _len = _ref.length; _i < _len; ++_i) {
@@ -422,61 +240,111 @@ async function initContext(args) {
         if ((typeof m === "function")) {
           ctx.use(m);
         } else {
-          throw Error(tpl("err: Context file export is not an array of functions or function {name}: {module}", {
-            name: ctxName,
-            module: m
-          }));
+          throw Error(`err: Context file export is not an array of functions or function ${ctxName}: ${m}`);
         }
       }
     } else {
-      throw Error(tpl("err: Context file export is not an array of functions or function {name}: {module}", {
-        name: ctxName,
-        module: module
-      }));
+      throw Error(`err: Context file export is not an array of functions or function ${ctxName}: ${m}`);
     }
   }
   return ctx;
 }
-initContext;
 async function main() {
-  var args, _ref;
+
+  let version = "0.0.0";
   try {
-    var args;
-    args = parseArgs(process.argv.slice(2));
-    if (args.help) {
-      showHelp();
-      process.exit(0);
-    }
-    if (args.version) {
-      try {
-        var pkg = require(path.resolve(__dirname, "../package.json"));
-        console.log(pkg.version || "0.0.0");
-      } catch (e) {
-        console.log("0.0.0");
-      }
-      process.exit(0);
-    }
-    validateArgs(args);
-    if (args.cmd === "rpc") {
-      await initConfig(args); // ensure config exists if needed
-      _ref = await runRpc(args);
-    } else if (args.cmd === "init") {
-      await initConfig(args);
-      _ref = null;
-    } else {
-      await initConfig(args); // auto-init if missing
-      _ref = await run(args);
-    }
+    var pkg = require(path.resolve(__dirname, "../package.json"));
+    version = pkg.version || "0.0.0"
   } catch (e) {
-    console.error(e.stack);
-    _ref = process.exit(1);
   }
-  return _ref;
+
+  const program = new Command();
+
+  program
+    .name("tune")
+    .description("Command Line Interface for Tune")
+    .version(version)
+    .helpOption(true)
+    .option("--home <dir>", "Tune config directory (default: ~/.tune)")
+    .option("--path <paths>", "Additional search paths (colon-separated)")
+    .addHelpText("after", "\nEXAMPLES:\n  tune --system \"You are Groot\" --user \"Hi how are you?\"\n  tune --user \"continue the conversation\" --filename chat.chat --save\n  tune --set test=hello --user \"@test\" --system \"Echo assistant\"\n  tune rpc\n  tune init --force\n");
+
+  program
+    .command("gen", { isDefault: true })
+    .description("start or continue ai conversation,\ndefault command")
+    .argument("[user]", "user message, the same as --user", (value) => value.replace(/\\n/g, "\n"))
+    .option("-u, --user <text>", "User message ", (value) => value.replace(/\\n/g, "\n"))
+    .option("-s, --system <text>", "System prompt to use", (value) => value.replace(/\\n/g, "\n"))
+    .option("-f, --filename <file>", "Chat file to load/save")
+    .option("--save", "Save conversation to file")
+    .addOption(new Option("--stop <mode>", "Stop condition", "assistant").choices(["assistant", "step"]).default("assistant"))
+    .option("--text <content>", "chat file content, overwrites system message", (value) => value.replace(/\\n/g, "\n"))
+    .addOption(new Option("-r, --response <type>", "response format").choices(["content", "json", "messages", "chat"]).default("content"))
+    .option("--silent", "generate response but do not print it")
+    .option("--set [params...]", "set a template variables, --set a=b --set c=d")
+    .action(async (user, opts, cmd) => {
+      opts = { ...opts, ...cmd.parent.opts() }
+      opts.user ||= user
+      opts.params = (opts.set || []).reduce((memo, item) => {
+        const [key, value] = item.split("=");
+        memo[key] = value
+        return memo
+      }, {})
+      // console.log("gen ", opts)
+      if (!opts.user && !opts.text && !opts.filename && !opts.system) {
+        return cmd.help();
+      }
+
+      await initConfig({ home: opts.home });
+      await run(opts);
+    })
+
+  program
+    .command("rpc")
+    .description("Start rpc server mode over stdio")
+    .option("-d, --debug [file]", "Enable debug output or write debug output to file")
+    .action(async (opts, cmd) => {
+      opts = { ...opts, ...cmd.parent.opts() }
+      // console.log(opts)
+      await initConfig(opts); // ensure config exists if needed
+      await runRpc(opts);
+    })
+
+  program
+    .command("ws")
+    .description("Start static webserver with tune context available browser side")
+    .option("-p, --port <port>", "port or socket to listen to", 8080)
+    .option("-s, --static", "should it serve static files like index.html etc", true)
+    .option("-r, --root <path>", "root directory for server", process.cwd())
+    .action( async (opts, cmd) => {
+      opts = { ...opts, ...cmd.parent.opts() }
+      // console.log(opts)
+      const { home, path, static, port, root } = opts
+      
+      await initConfig({ home });
+
+      const ctx = await initContext({ home, path });
+      ws({ port, static, root, ctx})
+    })
+
+  program
+    .command("init")
+    .description("Initialize Tune config directory")
+    .option("--force", "Force config initialization (with 'init')")
+    .action(async (opts, cmd) => {
+      opts = { ...opts, ...cmd.parent.opts() }
+      await initConfig(opts);
+    })
+
+  try {
+    await program.parseAsync(process.argv);
+  }catch (e) {
+    console.error(e.stack)
+    process.exit(1)
+  }
 }
-main;
 
-tpl;
-exports.parseArgs = parseArgs;
 exports.rpc = rpc;
 exports.main = main;
 exports.run = run;
+exports.ws = ws;

package/src/contextws.js

@@ -0,0 +1,201 @@
+// ContextWebsocket: explicit request/response wrapper over WebSocket.
+
+function assert(cond, msg) {
+  if (!cond) throw new Error(msg);
+}
+
+class StreamController {
+  constructor(onClose) {
+    this.queue = [];
+    this.waiter = null;
+    this.closed = false;
+    this.err = null;
+    this.onClose = onClose;
+  }
+
+  push(value, done = false) {
+    if (this.closed) return;
+    const items = [];
+    if (typeof value !== 'undefined') items.push({ value });
+    if (done) items.push({ done });
+
+    if (this.waiter && items.length) {
+      const { resolve } = this.waiter;
+      this.waiter = null;
+      resolve(items.shift());
+    }
+
+    this.queue.push(...items);
+    if (done) this.close();
+  }
+
+  close() {
+    if (this.closed) return;
+    this.closed = true;
+    if (this.waiter) {
+      const { resolve } = this.waiter;
+      this.waiter = null;
+      resolve({ value: undefined, done: true });
+    }
+    if (this.onClose) this.onClose();
+  }
+
+  fail(err) {
+    if (this.closed) return;
+    this.err = err;
+    this.closed = true;
+    if (this.waiter) {
+      const { reject } = this.waiter;
+      this.waiter = null;
+      reject(err);
+    }
+    if (this.onClose) this.onClose();
+  }
+
+  async next() {
+    if (this.err) throw this.err;
+    if (this.queue.length) return this.queue.shift();
+    if (this.closed) return { value: undefined, done: true };
+
+    return await new Promise((resolve, reject) => {
+      this.waiter = { resolve, reject };
+    });
+  }
+
+  [Symbol.asyncIterator]() {
+    return this;
+  }
+}
+
+function ContextWebsocket(url, { debug } = {}) {
+  assert(typeof WebSocket !== 'undefined', 'WebSocket is not available');
+  assert(typeof url !== 'undefined', 'url is not set');
+
+  socket = new WebSocket(url);
+  console.log(socket)
+
+  this.socket = socket;
+  this.callbacks = Object.create(null);
+  this.iterators = Object.create(null);
+  this.pending = [];
+  this.debug = typeof debug === 'function' ? debug : () => {};
+  this.ready = new Promise((resolve, reject) => {
+    if (socket.readyState === WebSocket.OPEN) return resolve();
+    socket.addEventListener('open', resolve, { once: true });
+    socket.addEventListener('error', reject, { once: true });
+  });
+
+
+  socket.addEventListener('message', (event) => this._onMessage(event));
+  socket.addEventListener('close', () => this._onClose());
+  socket.addEventListener('error', () => this._onClose());
+}
+
+ContextWebsocket.prototype._onClose = function () {
+  const err = new Error('websocket closed');
+  for (const id of Object.keys(this.callbacks)) {
+    this.callbacks[id].reject(err);
+    delete this.callbacks[id];
+  }
+  for (const id of Object.keys(this.iterators)) {
+    this.iterators[id].fail(err);
+    delete this.iterators[id];
+  }
+};
+
+ContextWebsocket.prototype._send = async function (payload) {
+  const data = JSON.stringify(payload);
+  this.debug('==>', data);
+  await this.ready;
+  this.socket.send(data);
+};
+
+ContextWebsocket.prototype._call = async function (method, params, stream) {
+  const id = Math.random().toString(36).slice(2);
+  await this._send({ id, method, args: params, stream: !!stream });
+
+  if (!stream) {
+    return await new Promise((resolve, reject) => {
+      this.callbacks[id] = { resolve, reject };
+    });
+  }
+
+  const ctrl = new StreamController(() => {
+    delete this.iterators[id];
+  });
+  this.iterators[id] = ctrl;
+  return ctrl;
+};
+
+ContextWebsocket.prototype._onMessage = function (event) {
+  let msg;
+  try {
+    msg = typeof event.data === 'string' ? JSON.parse(event.data) : event.data;
+  } catch {
+    return;
+  }
+
+  this.debug('<==', msg);
+  if (!msg || !msg.id) return;
+
+  const cb = this.callbacks[msg.id];
+  const iter = this.iterators[msg.id];
+
+  if (msg.error) {
+    const err = new Error(msg.error.message || 'rpc error');
+    err.stack = msg.error.stack;
+
+    if (cb) {
+      delete this.callbacks[msg.id];
+      cb.reject(err);
+      return;
+    }
+    if (iter) {
+      iter.fail(err);
+      return;
+    }
+    return;
+  }
+
+  if (cb && (Object.prototype.hasOwnProperty.call(msg, 'result') || msg.done)) {
+    delete this.callbacks[msg.id];
+    cb.resolve(msg.result);
+    return;
+  }
+
+  if (iter) {
+    if (msg.done && !Object.prototype.hasOwnProperty.call(msg, 'result')) {
+      iter.close();
+    } else {
+      iter.push(msg.result, !!msg.done);
+    }
+  }
+};
+
+ContextWebsocket.prototype.read = function (name, binary) {
+  return this._call('read', [name, binary], false);
+};
+
+ContextWebsocket.prototype.write = function (name, content) {
+  return this._call('write', [name, content], false);
+};
+
+ContextWebsocket.prototype.resolve = function (name, params) {
+  return this._call('resolve', [name, params], false);
+};
+
+ContextWebsocket.prototype.exec = function (name, params) {
+  return this._call('exec', [name, params], false);
+};
+
+ContextWebsocket.prototype.file2run = function (payload, params) {
+  return this._call('file2run', [payload, params], payload?.stream);
+};
+
+
+if (typeof module !== 'undefined' && module.exports) {
+  module.exports = ContextWebsocket;
+} else {
+  window.ContextWebsocket = ContextWebsocket;
+  window.ctx = new ContextWebsocket(window.location.origin.replace(/^http/, 'ws'));
+}

package/src/man.js

@@ -0,0 +1,117 @@
+const fs = require("fs")
+const path = require("path")
+// all the registered manuals
+// { name, description, filename/content } 
+
+let mans = [
+] 
+
+/*
+  @man include all manuals for all packages included
+  @man/ - list all the manuals, like list directory
+  @man/tune - get manual for tune core package 
+  @man/tune-basic-toolset - get manual for tune-basic-tool set package
+
+  TODO:
+  @man/tune/js-api - lis
+  @man/tune/agents - lis
+*/
+
+function read({ filename, content }) {
+  if (content) {
+    return content
+  }
+  if (!fs.existsSync(filename)) {
+    return `file ${filename} not found`
+  }
+  return fs.readFileSync(filename)
+}
+
+module.exports = ({ mount } = {}) => async (name, { type }) => {
+  if ((type || "any") !== 'any' && type !== 'text' ) {
+    return
+  }
+  mount ||= "man"
+
+  // supported only with mount
+  if (!mount) {
+    return 
+  }
+
+  // @man include all the manuals
+  if (name === mount) {
+    return {
+      type: "text",
+      name,
+      read: async () => mans.map(man => `<${man.name}>\n ${read(man)} \n</${man.name}>`).join("\n")
+    }
+  }
+
+  if (!name.startsWith(mount + '/')) {
+    return 
+  }
+
+  // @man/ - list all the manuals to include
+  if (name === mount + '/') {
+    return {
+      type: "text",
+      name,
+      read: async () => mans.map(man => `${man.name} - ${man.description}`).join("\n")
+    }
+  }
+
+
+  // @man/tune - get manual for tune core package 
+  // @man/tune-basic-toolset - get manual for tune-basic-tool set package
+  
+  const actualName = name.slice(mount.length + 1);
+
+  const man = mans.find(man => man.name === actualName);
+  if (!man) return
+
+  return {
+    type: "text",
+    name,
+    read: async() => read(man.filename)
+  }
+}
+
+const addManual = (dirname) => {
+  let package = path.resolve(dirname, "package.json");
+  if (!fs.existsSync(package)){
+    dirname = path.resolve(dirname, "..")
+    package = path.resolve(dirname, "package.json");
+  }
+
+  if (!fs.existsSync(package)){
+    throw Error(`cant make manual ${package} not found`);
+  }
+
+  const { name, version, description } = JSON.parse(fs.readFileSync(package, "utf8"));
+
+  const newMans = mans.filter(man => man.name !== name)
+
+  const filename = path.resolve(dirname, "README.md")
+
+  newMans.push({name, description, filename})
+  mans = newMans
+}
+
+addManual(__dirname)
+/*
+const addManual = ({name, description, content, filename}) => {
+  const  = params;
+
+  const newMans = mans.filter(man => man.name !== name)
+  if (!name) {
+    throw Error(`name is not set for manual`)
+  }
+  if (!content && !filename) {
+    throw Error(`neither content nor filename is set for '${name}' manual`)
+  }
+
+  newMans.push({name, description, content, filename})
+}
+*/
+
+module.exports.addManual = addManual;

package/src/ws.js

@@ -0,0 +1,123 @@
+const http = require('http');
+const fs = require('fs');
+const path = require('path');
+const WebSocket = require('ws');
+const mime = require('mime-types');
+
+function makeServer({ port, static, root, ctx }) {
+  root = path.join(root || process.cwd());
+
+  function sendFile(filePath, res){ 
+    fs.readFile(filePath, (err, data) => {
+      if (err) {
+        res.writeHead(404, { 'Content-Type': 'text/plain' });
+        return res.end('404 Not Found');
+      }
+      res.writeHead(200, {
+        'Content-Type': mime.lookup(filePath) || 'application/octet-stream',
+      });
+      res.end(data);
+    });
+  }
+  const server = http.createServer((req, res) => {
+    console.log(`[${req.method.toUpperCase()}] ${req.url}`)
+    if (req.method.toUpperCase() !== "GET") { 
+      res.writeHead(200, { 'Content-Type': 'text/plain' });
+      return res.end("ok");
+    }
+    if (req.url === "/contextws.js") {
+      return sendFile(path.join(__dirname,  "contextws.js"), res)
+    }
+    if (!static) {
+      res.writeHead(200, { 'Content-Type': 'text/plain' });
+      return res.end("ok");
+    }
+
+    let filePath = req.url === '/' ? '/index.html' : req.url;
+    // TODO: check if resolved path is not out of root 
+    filePath = path.join(root, filePath);
+    sendFile(filePath, res)
+  });
+
+  const wss = new WebSocket.Server({ server });
+
+  wss.on('connection', (ws) => {
+    ws.on('message', async (msg) => {
+      let lctx = ctx.clone()
+      let { id, method, args, ...rest } = JSON.parse(msg.toString());
+      let result = { id }
+      if (method === "read") {
+        const [ name, binary ] = args
+        let content = await lctx.read(name, binary)
+        if (binary) {
+          content = content.toString("base64");
+        }
+        result = { id, result: content, binary, done: true }
+      } else if (method === "write") {
+        let [ name, content ] = args;
+        const { binary } = rest
+        if (binary) {
+          content = Buffer.from(content, "base64")
+        }
+        await lctx.write(name, content)
+        result.done = true
+      } else if (method === "exec") {
+        const [name, params] = args; 
+        const node  = await lctx.resolve(name)
+        if (!node) {
+          result.error = { message: `${name} not found` }
+        } else if (!node.exec) {
+          result.error = { message: `${name} not executable` }
+        } else {
+          const content = await ctx.exec(name, params)
+          result.result = content
+          result.done = true
+        }
+      } else if (method === "resolve") {
+        const [name, params] = args; 
+        const content = await lctx.resolve(name, params)
+        result.result = content
+        result.done = true
+      } else if (method === "file2run") {
+        const [ payload, params ]  = args
+
+        const r = await lctx.file2run({ ...payload, errors: "message" }, params);
+        if (!payload?.stream) {
+          result.result = r
+          result.done = true
+        } else {
+          (async () => {
+            try {
+              let chunk = {};
+              let lastRes
+              while(!chunk.done) {
+                chunk = await r.next();
+                result.result = (chunk.value || "");
+                ws.send(JSON.stringify(result));
+              }
+              result.done = true
+              ws.send(JSON.stringify(result));
+            } catch (e) {
+              delete result.result
+              result.error = {message: e.message, stack: e.stack}
+              ws.send(JSON.stringify(result));
+            }
+          })()
+        }
+      } else {
+        result.error = { message: `method not found ${method}`}
+      }
+      ws.send(JSON.stringify(result));
+    });
+  });
+
+  server.listen(port, () => {
+    if (Number.isNaN(parseInt(port))) {
+      console.log(`listening ${port}`);
+    } else {
+      console.log(`listening http://localhost:${port}`);
+    }
+  });
+}
+
+module.exports = makeServer