tune-sdk 0.1.5 → 0.1.8

package/bin/cli.js

@@ -20,26 +20,6 @@ process.argv.slice(2).forEach(arg => {
   }
 });
 
-// Example usage outputs (replace with your real logic)
-/*
-if (args.filename) {
-  console.log('filename:', args.filename);
-}
-if (args.system) {
-  console.log('system:', args.system);
-}
-if (args.user) {
-  console.log('user:', args.user);
-}
-if ('save' in args) {
-  console.log('save flag set');
-}
-if (args.stop) {
-  console.log('stop:', args.stop);
-}
-*/
-
-// Place your main script/logic here
 const tune = require('../dist/tune.js');
 const { fsctx } = require('../dist/fsctx.js');
 const fs = require("fs"); 
@@ -105,9 +85,19 @@ async function main() {
       throw(Error(`err: Context file export is not an array of functions or function ${filename}`))
     }
   }
+  let stopVal = args.stop || "assistant";
+  if (stopVal !== "step" && stopVal != "assistant") {
+    stopVal = function(msgs) {
+      if (!msgs.length) return false
+      const lastMsg = msgs[msgs.length - 1]
+      if (!lastMsg.content) return false
+      return (lastMsg.content.indexOf(args.stop) !== -1)
+    }
+  }
+
+  let res = await tune.text2run(chat, ctx, { stop: stopVal })
 
   const longFormatRegex = /^(system|user|tool_call|tool_result|assistant|error):/;
-  let res = await tune.text2run(chat, ctx, { stop: args.stop || "assistant" })
   res = tune.msg2text(res, longFormatRegex.test(chat))
   if (args.filename && args.save) {
     chat += "\n" + res 

package/dist/fsctx.js

@@ -1,4 +1,4 @@
-var path, tune, makeContext, envmd, TuneError, text2run;
+var path, tune, envmd, TuneError, text2run;
 
 function extend() {
   var _i;
@@ -616,7 +616,6 @@ if (typeof window !== "undefined") {
 }
 path = require("path");
 tune = require("./tune");
-makeContext = tune.makeContext;
 envmd = tune.envmd;
 TuneError = tune.TuneError;
 text2run = tune.text2run;
@@ -669,9 +668,10 @@ async function runFile(filename, ctx) {
       text = await node.read();
       var lctx;
       lctx = ctx.clone();
+      lctx.stack.push(node);
       lctx.ms.unshift(envmd(args[0]));
       var res;
-      res = await text2run(text, lctx, {
+      res = await lctx.text2run(text, {
         stop: "assistant"
       });
       _ref = res["slice"](-1)[0].content.replace(/@/g, "\\@");
@@ -767,15 +767,21 @@ function fsctx(paths, opts, fs) {
     }));
 
   function mkfsmd1(p) {
-    return (async function(name, ctx, args) {
-      var fname, parsed, item, parsed1, fileType, fullname, schemaFile, schema, _i, _ref, _len, _ref0, _ref1, _ref2, _ref3;
+    async function fsmd(name, args) {
+      var self, result, parsed, dir, re, item, parsed1, fileType, fullname, schemaFile, schema, res, _i, _ref, _len, _ref0, _ref1, _ref2, _ref3;
+      var self;
+      var result;
+      self = this;
+      result = ((args.output === "all") ? [] : undefined);
       if (!name) return;
-      var fname;
       var parsed;
-      fname = path.resolve(p, ((name === "*") ? "default" : name));
-      parsed = pparse(fname);
-      if (!fs.existsSync(parsed.dir)) return;
-      _ref = fs.readdirSync(parsed.dir)
+      var dir;
+      var re;
+      parsed = ((args.match === "exact") ? pparse(path.resolve(p, name)) : undefined);
+      dir = (((typeof parsed !== "undefined") && (parsed !== null) && !Number.isNaN(parsed) && (typeof parsed.dir !== "undefined") && (parsed.dir !== null) && !Number.isNaN(parsed.dir)) ? parsed.dir : (((typeof p !== "undefined") && (p !== null) && !Number.isNaN(p)) ? p : undefined));
+      re = ((args.match === "regex") ? new RegExp(name) : undefined);
+      if (!fs.existsSync(dir)) return;
+      _ref = fs.readdirSync(dir)
         .sort((function(a, b) {
           var exts, idx1, idx2;
           var exts;
@@ -789,8 +795,9 @@ function fsctx(paths, opts, fs) {
       for (_i = 0, _len = _ref.length; _i < _len; ++_i) {
         item = _ref[_i];
         var parsed1;
-        parsed1 = pparse(item);
-        if ((parsed1.name !== parsed.name)) continue;
+        parsed1 = pparse(path.join(dir, item));
+        if (((args.match === "exact") && (parsed.base !== item) && (parsed.base !== (parsed1.name + parsed1.ext2)) && (parsed.base !== parsed1.name))) continue;
+        if ((re && !re.test(item))) continue;
         var fileType;
         if ((parsed1.ext2 === ".tool")) {
           _ref0 = "tool";
@@ -809,166 +816,172 @@ function fsctx(paths, opts, fs) {
         }
         fileType = _ref0;
         var fullname;
-        fullname = path.resolve(parsed.dir, item);
-        if ((args && (args !== fileType))) continue;
-        if (((item === parsed.base) || !parsed.ext || (parsed1.ext2 && (parsed.base === (parsed1.name + parsed1.ext2))) || (name === "*"))) {
-          switch (fileType) {
-            case "tool":
-              var schemaFile;
-              schemaFile = path.format({
-                root: parsed.root,
-                dir: parsed.dir,
-                name: parsed1.name,
-                ext: ".schema.json"
-              });
-              var schema;
-              schema;
-              if (fs.existsSync(schemaFile)) {
-                schema = JSON.parse(fs.readFileSync(schemaFile, "utf8"));
-              } else if (opts && opts.makeSchema) {
-                schema = await opts.makeSchema({
-                  text: fs.readFileSync(fullname, "utf8")
-                }, ctx);
-                fs.writeFileSync(schemaFile, schema);
-                schema = JSON.parse(schema);
-              } else {
-                throw new TuneError(("schema file not found " + schemaFile));
-              }
-              _ref1 = {
-                type: "tool",
-                schema: schema,
-                name: parsed1.name,
-                exec: (async function(params, ctx) {
-                  return runFile(fullname, ctx, params);
-                }),
-                read: (async function() {
-                  return fs.readFileSync(fullname, "utf8");
-                }),
-                dirname: parsed.dir,
-                fullname: fullname
-              }
-              break;
-            case "llm":
-              _ref1 = {
-                type: "llm",
-                dirname: parsed.dir,
-                fullname: fullname,
-                name: parsed1.name,
-                exec: (async function(payload, ctx) {
-                  return runFile(fullname, ctx, payload);
-                }),
-                read: (async function() {
-                  return fs.readFileSync(fullname, "utf8");
-                })
-              }
-              break;
-            case "context":
-              ctx.use((async function(name, ctx, args, next) {
-                return runFile(fullname, ctx, name, ctx, args, next);
-              }));
-              _ref1 = {
-                type: "text",
-                fullname: fullname,
-                name: parsed1.name,
-                dirname: parsed.dir,
-                read: (async function() {
-                  return "";
-                })
-              }
-              break;
-            case "processor":
-              _ref1 = {
-                type: "processor",
-                name: parsed1.name,
-                exec: (function(node, args, ctx) {
-                  return runFile(fullname, ctx, node, args);
-                }),
-                read: (async function() {
-                  return fs.readFileSync(fullname, "utf8");
-                }),
-                dirname: parsed.dir,
-                fullname: fullname
-              }
-              break;
-            case "image":
-              if ((parsed1.ext === ".jpg" || parsed1.ext === ".jpeg")) {
-                _ref2 = "image/jpeg";
-              } else if (parsed1.ext === ".png") {
-                _ref2 = "image/png";
-              } else if (parsed1.ext === ".webp") {
-                _ref2 = "image/webp";
-              } else {
-                _ref2 = undefined;
-              }
-              _ref1 = {
-                type: "image",
-                dirname: parsed.dir,
-                fullname: fullname,
-                mimetype: _ref2,
-                read: (async function() {
-                  return fs.readFileSync(fullname);
-                })
-              }
-              break;
-            case "audio":
-              if ((parsed1.ext === ".mp3")) {
-                _ref3 = "audio/mpeg";
-              } else if (parsed1.ext === ".wav") {
-                _ref3 = "audio/wav";
-              } else {
-                _ref3 = undefined;
-              }
-              _ref1 = {
-                type: "audio",
-                dirname: parsed.dir,
-                fullname: fullname,
-                mimetype: _ref3,
-                read: (async function() {
-                  return fs.readFileSync(fullname);
-                })
-              }
-              break;
-            case "text":
-              _ref1 = {
-                type: "text",
-                dirname: parsed.dir,
-                fullname: fullname,
-                name: parsed1.name,
-                read: (async function(binary) {
-                  var stat, buf, i, c;
-                  var stat;
-                  stat = fs.lstatSync(fullname);
-                  if (stat.isDirectory()) return fs.readdirSync(fullname);
-                  if (binary) return fs.readFileSync(fullname);
-                  var buf;
-                  buf = fs.readFileSync(fullname, "utf8");
-                  var i;
-                  i = 0;
-                  while (i < Math.min(1024, buf.length)) {
-                    var c;
-                    c = buf.charCodeAt(i);
-                    if (((c === 65533) || (c <= 8))) throw Error(tpl("{} is a binary file, can not include it", fullname));
-                    i++;
-                  }
-                  return buf;
-                })
-              }
-              break;
-            default:
-              _ref1 = undefined;
-          }
-          return _ref1;
+        fullname = path.resolve(dir, item);
+        if (((args.type !== "any") && (args.type !== fileType))) continue;
+        var res;
+        switch (fileType) {
+          case "tool":
+            var schemaFile;
+            schemaFile = path.format({
+              root: parsed1.root,
+              dir: parsed1.dir,
+              name: parsed1.name,
+              ext: ".schema.json"
+            });
+            var schema;
+            schema;
+            if (fs.existsSync(schemaFile)) {
+              schema = JSON.parse(fs.readFileSync(schemaFile, "utf8"));
+            } else if (opts && opts.makeSchema && (opts.output !== "all")) {
+              schema = await opts.makeSchema({
+                text: fs.readFileSync(fullname, "utf8")
+              }, self);
+              fs.writeFileSync(schemaFile, schema);
+              schema = JSON.parse(schema);
+            } else {
+              throw new Error(("schema file not found " + schemaFile));
+            }
+            _ref1 = {
+              type: "tool",
+              schema: schema,
+              name: parsed1.name,
+              exec: (async function(params, ctx) {
+                return runFile(fullname, ctx, params);
+              }),
+              read: (async function() {
+                return fs.readFileSync(fullname, "utf8");
+              }),
+              dirname: parsed1.dir,
+              fullname: fullname
+            }
+            break;
+          case "llm":
+            _ref1 = {
+              type: "llm",
+              dirname: parsed1.dir,
+              fullname: fullname,
+              name: parsed1.name,
+              exec: (async function(payload, ctx) {
+                return runFile(fullname, ctx, payload);
+              }),
+              read: (async function() {
+                return fs.readFileSync(fullname, "utf8");
+              })
+            }
+            break;
+          case "context":
+            if ((args.output !== "all")) self.use((async function(name, args) {
+              return runFile(fullname, this, name, args);
+            }));
+            _ref1 = {
+              type: "text",
+              fullname: fullname,
+              name: parsed1.name,
+              dirname: parsed1.dir,
+              read: (async function() {
+                return "";
+              })
+            }
+            break;
+          case "processor":
+            _ref1 = {
+              type: "processor",
+              name: parsed1.name,
+              exec: (function(node, args, ctx) {
+                return runFile(fullname, ctx, node, args);
+              }),
+              read: (async function() {
+                return fs.readFileSync(fullname, "utf8");
+              }),
+              dirname: parsed1.dir,
+              fullname: fullname
+            }
+            break;
+          case "image":
+            if ((parsed1.ext === ".jpg" || parsed1.ext === ".jpeg")) {
+              _ref2 = "image/jpeg";
+            } else if (parsed1.ext === ".png") {
+              _ref2 = "image/png";
+            } else if (parsed1.ext === ".webp") {
+              _ref2 = "image/webp";
+            } else {
+              _ref2 = undefined;
+            }
+            _ref1 = {
+              type: "image",
+              dirname: parsed1.dir,
+              fullname: fullname,
+              mimetype: _ref2,
+              read: (async function() {
+                return fs.readFileSync(fullname);
+              })
+            }
+            break;
+          case "audio":
+            if ((parsed1.ext === ".mp3")) {
+              _ref3 = "audio/mpeg";
+            } else if (parsed1.ext === ".wav") {
+              _ref3 = "audio/wav";
+            } else {
+              _ref3 = undefined;
+            }
+            _ref1 = {
+              type: "audio",
+              dirname: parsed1.dir,
+              fullname: fullname,
+              mimetype: _ref3,
+              read: (async function() {
+                return fs.readFileSync(fullname);
+              })
+            }
+            break;
+          case "text":
+            _ref1 = {
+              type: "text",
+              dirname: parsed1.dir,
+              fullname: fullname,
+              name: parsed1.name,
+              read: (async function(binary) {
+                var stat, buf, i, c;
+                var stat;
+                stat = fs.lstatSync(fullname);
+                if (stat.isDirectory()) return fs.readdirSync(fullname);
+                if (binary) return fs.readFileSync(fullname);
+                var buf;
+                buf = fs.readFileSync(fullname, "utf8");
+                var i;
+                i = 0;
+                while (i < Math.min(1024, buf.length)) {
+                  var c;
+                  c = buf.charCodeAt(i);
+                  if (((c === 65533) || (c <= 8))) throw Error(tpl("{} is a binary file, can not include it", fullname));
+                  i++;
+                }
+                return buf;
+              })
+            }
+            break;
+          default:
+            _ref1 = undefined;
+        }
+        res = _ref1;
+        if ((args.output === "all")) {
+          result.push(res);
+        } else {
+          result = res;
+          break;
         }
       }
-      return undefined;
-    });
+      return result;
+    }
+    return fsmd;
   }
   mkfsmd1;
   envCache = {};
-  return (async function(name, ctx, args, next) {
-    var lpaths, handles, p, envFile, res, _i, _ref, _len;
+  return (async function(name, args) {
+    var lpaths, handles, result, p, envFile, handle, res, _i, _ref, _len;
     var lpaths;
-    lpaths = ctx.stack
+    lpaths = this.stack
       .filter((function(item) {
         return !!item.dirname;
       }))
@@ -979,6 +992,8 @@ function fsctx(paths, opts, fs) {
       .concat(paths);
     var handles;
     handles = [];
+    var result;
+    result = ((args.output === "all") ? [] : undefined);
     _ref = lpaths;
     for (_i = 0, _len = _ref.length; _i < _len; ++_i) {
       p = _ref[_i];
@@ -994,11 +1009,19 @@ function fsctx(paths, opts, fs) {
       handles.push(mkfsmd1(p));
     }
     while (handles.length) {
+      var handle;
+      handle = handles.shift();
       var res;
-      res = await handles.shift()(name, ctx, args);
-      if (res) return res;
+      res = await handle.call(this, name, args);
+      if (!res) continue;
+      if ((args.output === "all")) {
+        Array.isArray(res) ? result = result.concat(res) : result.push();
+      } else {
+        result = res;
+        break;
+      }
     }
-    return next();
+    return result;
   });
 }
 fsctx;
@@ -1012,7 +1035,7 @@ function tpl(str) {
       _ref = str.replace(/{(\W*)(\w*)(\W*)}/gm, (function(_, pre, name, post) {
         return (function(res) {
           paramIndex += 1;
-          return (res ? ((pre || "") + res + (post || "")) : "");
+          return ((typeof res !== 'undefined') ? ((pre || "") + res + (post || "")) : "");
         })(params[name || paramIndex]);
       }));
     } catch (e) {

package/dist/tune.js

@@ -19,6 +19,26 @@ function extend() {
 }
 extend;
 
+function AsyncIter() {
+  return (this[Symbol.asyncIterator] = this);
+}
+AsyncIter;
+AsyncIter.prototype.next = (async function() {
+  var self, result;
+  var self;
+  self = this;
+  await _once((function() {
+    return (!!self.err || !!self.result);
+  }), (function() {
+    return undefined;
+  }));
+  var result;
+  result = self.result;
+  self.result = undefined;
+  if (self.err) throw self.err;
+  return result;
+});
+
 function TuneError(message, filename, row, col, stack, originalError) {
   var lastItem;
   var stack;
@@ -34,7 +54,7 @@ function TuneError(message, filename, row, col, stack, originalError) {
   this.message = message;
   this.row = row;
   this.col = col;
-  this.stack = stack;
+  this._stack = stack;
   this.error = originalError;
   return this;
 }
@@ -42,7 +62,7 @@ TuneError;
 TuneError.prototype = Object.create(Error.prototype);
 TuneError.prototype.constructor = TuneError;
 TuneError.wrap = (function(err, filename, row, col) {
-  return ((err.name === "TuneError") ? new TuneError(err.message, filename, row, col, err.stack, err.error) : new TuneError("Internal Error", filename, row, col, undefined, err));
+  return ((err.name === "TuneError") ? new TuneError(err.message, filename, row, col, err._stack, err.error) : new TuneError("Internal Error", filename, row, col, undefined, err));
 });
 TuneError.ctx2stack = (function(ctx) {
   return ctx.stack.map((function(item) {
@@ -54,11 +74,16 @@ TuneError.ctx2stack = (function(ctx) {
   }));
 });
 TuneError.prototype.toString = (function() {
-  return (this.message + "\n" + (this.stack || [])
-    .map((function() {
-      return ("    at " + arguments[0].filename + (((typeof arguments !== 'undefined') && (typeof arguments[0] !== 'undefined') && (typeof arguments[0].row !== 'undefined')) ? (":" + arguments[0].row) : "") + (((typeof arguments !== 'undefined') && (typeof arguments[0] !== 'undefined') && (typeof arguments[0].col !== 'undefined')) ? (":" + arguments[0].col) : ""));
-    }))
-    .join("\n") + (this.error ? (this.error.stack ? ("\n" + this.error.stack) : this.error.message) : ""));
+  return this.stack;
+});
+Object.defineProperty(TuneError.prototype, "stack", {
+  get: (function() {
+    return ("TuneError: " + this.message + "\n" + (this._stack || [])
+      .map((function() {
+        return ("    at " + arguments[0].filename + (((typeof arguments !== 'undefined') && (typeof arguments[0] !== 'undefined') && (typeof arguments[0].row !== 'undefined')) ? (":" + arguments[0].row) : "") + (((typeof arguments !== 'undefined') && (typeof arguments[0] !== 'undefined') && (typeof arguments[0].col !== 'undefined')) ? (":" + arguments[0].col) : ""));
+      }))
+      .join("\n") + (this.error ? (this.error.stack ? ("\n" + this.error.stack) : this.error.message) : ""));
+  })
 });
 var dJSON = (() => {
   var de = Object.create;
@@ -1017,6 +1042,7 @@ function Context() {
     return (item.name === "write");
   }));
   this.stack = [];
+  this.type = "Context";
   return this;
 }
 Context;
@@ -1033,24 +1059,55 @@ Context.prototype.clone = (function() {
 Context.prototype.use = (function(middleware) {
   return ((middleware.name === "write") ? this.ws.push(middleware) : this.ms.push(middleware));
 });
-Context.prototype.resolve = (async function(name, args, ms) {
+Context.prototype.resolve = (async function(name, args) {
+  var i, output, match, type, result, res, md;
+  if (!this.ms.length) return;
+  var i;
+  var output;
+  var match;
+  var type;
+  var result;
   var res;
-  ms = ms || this.ms;
-  if (!ms.length) return;
-  var res;
-  res = ms[0](name, this, args, this.resolve.bind(this, name, args, ms.slice(1)));
-  res = await res;
-  if (!res) return res;
-  if ((typeof res !== "object")) throw Error(tpl("resolved '@{name}' node must be 'object' but got '{res}'", {
-    name: name,
-    res: typeof res
-  }));
-  if (!res.type) throw Error(tpl("resolved '@{name}' node must have a 'type' property", {
-    name: name
-  }));
-  return res;
+  i = 0;
+  output = (((typeof args !== "undefined") && (args !== null) && !Number.isNaN(args) && (typeof args.output !== "undefined") && (args.output !== null) && !Number.isNaN(args.output)) ? args.output : (((typeof "first" !== "undefined") && ("first" !== null) && !Number.isNaN("first")) ? "first" : undefined));
+  match = (((typeof args !== "undefined") && (args !== null) && !Number.isNaN(args) && (typeof args.match !== "undefined") && (args.match !== null) && !Number.isNaN(args.match)) ? args.match : (((typeof "exact" !== "undefined") && ("exact" !== null) && !Number.isNaN("exact")) ? "exact" : undefined));
+  type = (((typeof args !== "undefined") && (args !== null) && !Number.isNaN(args) && (typeof args.type !== "undefined") && (args.type !== null) && !Number.isNaN(args.type)) ? args.type : (((typeof "any" !== "undefined") && ("any" !== null) && !Number.isNaN("any")) ? "any" : undefined));
+  result = ((output === "all") ? [] : undefined);
+  res = undefined;
+  while (i < this.ms.length) {
+    var md;
+    md = this.ms[i];
+    res = await md.call(this, name, {
+      output: output,
+      match: match,
+      type: type
+    });
+    if (!res) {
+      i++;
+      continue;
+    }
+    if (((output === "all") && Array.isArray(res))) {
+      result = result.concat(res);
+      i++;
+      continue;
+    }
+    if ((typeof res !== "object")) throw Error(tpl("resolved '@{name}' node must be 'object' but got '{res}'", {
+      name: name,
+      res: typeof res
+    }));
+    if (!res.type) throw Error(tpl("resolved '@{name}' node must have a 'type' property", {
+      name: name
+    }));
+    if ((output === "first")) {
+      result = res;
+      break;
+    }
+    result.push(res);
+    i++;
+  }
+  return result;
 });
-Context.prototype.text2run = (async function(text, opts) {
+Context.prototype.text2run = (function(text, opts) {
   return text2run(text, this, opts);
 });
 Context.prototype.text2ast = (async function(text) {
@@ -1080,7 +1137,7 @@ Context.prototype.exec = (async function(name, args) {
   var resolved;
   resolved = await this.resolve(name);
   if ((resolved && resolved.exec)) {
-    return _ref = await resolved.exec(args);
+    return _ref = await resolved.exec.call(this, args, this);
   } else {
     _ref = undefined;
   }
@@ -1103,49 +1160,49 @@ function envmd(md) {
       if ((typeof md[name] === "string" || typeof md[name] === "number" || typeof md[name] === "boolean")) {
         _ref = (lmd[name] = {
           type: "text",
+          name: name,
           read: (function() {
             return md[name];
           })
         });
       } else if (typeof md[name] === "object") {
-        _ref = (lmd[name] = md[name]);
+        lmd[name] = md[name];
+        _ref = (lmd[name].name = name);
       } else {
         _ref = undefined;
         throw new TuneError(("unsupported type of value '" + name + "': " + util.inspect(md)));
       }
       return _ref;
     }));
-  return (async function(name, ctx, args, next) {
-    var llmd, key, val, firstKey, _ref, _len, _ref0;
+  async function hash(name, args) {
+    var llmd, result, re, key, val, _ref, _len;
     var llmd;
+    var result;
+    var re;
     llmd = {};
+    result = ((args.output === "all") ? [] : undefined);
+    re = ((args.match === "regex") ? new RegExp(name) : undefined);
     _ref = lmd;
     for (key in _ref) {
       val = _ref[key];
-      if ((args && (args === val.type))) {
-        llmd[key] = val;
-      } else if (!args) {
-        llmd[key] = val;
+      if (((args.type !== val.type) && (args.type !== "any"))) continue;
+      if (((args.match === "exact") && (name !== key))) continue;
+      if ((re && !re.test(key))) continue;
+      if ((args.output === "first")) {
+        result = Object.assign({}, val);
+        break;
       }
+      result.push(Object.assign({}, val));
     }
-    var firstKey;
-    firstKey = Object.keys(llmd)[0];
-    if ((firstKey && (name === "*"))) {
-      _ref0 = llmd[firstKey];
-    } else if (llmd.hasOwnProperty(name)) {
-      _ref0 = llmd[name];
-    } else {
-      _ref0 = next();
-    }
-    return _ref0;
-  });
+    return result;
+  }
+  return hash;
 }
 envmd;
 
 function makeContext() {
   var ctx, _i;
   var args = 1 <= arguments.length ? [].slice.call(arguments, 0, _i = arguments.length - 0) : (_i = 0, []);
-  if ((args[0] instanceof Context)) return args[0];
   var ctx;
   ctx = new Context();
   if (!args[0]) return ctx;
@@ -1353,25 +1410,26 @@ async function text2payload(text, ctx) {
   return await ast2payload(ast);
 }
 text2payload;
-async function text2ast(text, ctx) {
+async function text2ast(text, ctx, resolve) {
   var lastChar, lastRole;
   var lastChar;
   lastChar = '';
   var lastRole;
   lastRole = '';
   async function parse(text, recursive, ctx) {
-    var nodeStack, re, nodes, index, m, match, role, roleName, name, prefix, args, proc, row, col, filename, resolved, pargs, pname, p, err, schema, lctx, _ref, _i, _i0, _ref0, _len;
-    ctx = makeContext(ctx);
+    var nodeStack, nodes, index, reAst, m, row, col, match, role, roleName, name, prefix, args, proc, filename, resolved, pargs, pname, p, err, schema, lctx, _ref, _i, _i0, _ref0, _len;
     nodeStack = nodeStack || [];
-    var re;
-    re = /(?<prefix1>@{1,2})\{(?<name1>(?:\\\}|[^}])+)\}|(?<prefix>@{1,2})(?<name>[\S]+)|(?<role>^(?:s|system|u|user|a|assistant|tc|tool_call|tr|tool_result|c|comment|au|audio|err|error))\s*(?<roleName>\([^\)]+\))?\s*:/gm;
     var nodes;
     nodes = [];
     var index;
     index = 0;
-    while (m = re.exec(text)) {
+    reAst = /(?<prefix1>@{1,2})\{(?<name1>(?:\\\}|[^}])+)\}|(?<prefix>@{1,2})(?<name>[\S]+)|(?<role>^(?:s|system|u|user|a|assistant|tc|tool_call|tr|tool_result|c|comment|au|audio|err|error))\s*(?<roleName>\([^\)]+\))?\s*:/gm;
+    while (m = reAst.exec(text)) {
       if (((lastRole === "a" || lastRole === "tc" || lastRole === "err" || lastRole === "assistant" || lastRole === "tool_call" || lastRole === "error") && !m.groups.role)) continue;
       if (((m.index > 0) && (text.charAt(m.index - 1) === String.fromCharCode(92)))) continue;
+      _ref = getPos(m.index, text);
+      row = _ref[0];
+      col = _ref[1];
       if ((m.index - index)) {
         nodes.push({
           type: "text",
@@ -1379,6 +1437,7 @@ async function text2ast(text, ctx) {
           end: m.index,
           value: text.slice(index, m.index)
         });
+        index = m.index;
         lastChar = text.charAt(m.index - 1);
       }
       var match;
@@ -1391,7 +1450,7 @@ async function text2ast(text, ctx) {
         nodes.push({
           type: ((lastChar === '' || lastChar === "\n") ? "role" : "text"),
           start: index,
-          end: m.index,
+          end: m.index + match.length,
           role: role,
           roleName: (roleName ? roleName.slice(1, -1) : undefined),
           value: match
@@ -1409,7 +1468,16 @@ async function text2ast(text, ctx) {
       prefix = m.groups.prefix || m.groups.prefix1;
       args = undefined;
       proc = undefined;
-      (function(it) {
+      if (!ctx) {
+        nodes.push({
+          type: "var",
+          start: m.index,
+          end: m.index + match.length,
+          value: match
+        });
+        index = m.index + match.length;
+        continue;
+      }(function(it) {
         it = it.replace(/\\\}/g, "}");
         it = it.split("|");
         it = it.map((function(it) {
@@ -1427,9 +1495,6 @@ async function text2ast(text, ctx) {
         it = (proc = ((it.length > 1) ? it.slice(1) : undefined));
         return it;
       })(name);
-      _ref = getPos(m.index, text);
-      row = _ref[0];
-      col = _ref[1];
       var filename;
       filename = (function(it) {
         it = it["slice"](-1)[0];
@@ -1443,15 +1508,17 @@ async function text2ast(text, ctx) {
           var pname;
           pname = pargs.shift();
           var p;
-          p = await ctx.resolve(pname, "processor");
+          p = await ctx.resolve(pname, {
+            type: "processor"
+          });
           if (!p) throw new TuneError(("'" + pname + "' processor not found"), filename, row, col);
           if ((typeof p.exec !== "function")) throw new TuneError(("'" + pname + "' does not have exec function"), filename, row, col);
           try {
-            resolved = await p.exec(resolved, pargs[0], ctx);
+            resolved = await p.exec.call(ctx, resolved, pargs[0], ctx);
           } catch (e) {
             var err;
             err = new TuneError(e.message, (p.fullname || p.name), undefined, undefined, [], e);
-            err.stack.push({
+            err._stack.push({
               filename: filename,
               row: row,
               col: col
@@ -1491,7 +1558,7 @@ async function text2ast(text, ctx) {
           lctx = ctx.clone();
           lctx.stack.push(resolved);
           try {
-            resolved.nodes = await parse(await resolved.read(), true, lctx);
+            resolved.nodes = await parse(resolved.value, true, lctx);
           } catch (e) {
             throw TuneError.wrap(e, filename, row, col);
           }
@@ -1756,11 +1823,12 @@ function ast2payload(ast) {
 ast2payload;
 async function payload2http(payload, ctx) {
   var llm, stack, lastStack, body, _ref;
-  ctx = makeContext(ctx);
   var llm;
   llm = payload.llm;
   delete payload.llm;
-  if (!llm) llm = await ctx.resolve("*", "llm");
+  if (!llm) llm = await ctx.resolve("default", {
+    type: "llm"
+  });
   if (!llm) {
     var stack;
     stack = TuneError.ctx2stack(ctx);
@@ -1777,7 +1845,7 @@ async function payload2http(payload, ctx) {
     }
   }));
   try {
-    _ref = await llm.exec(body, ctx);
+    _ref = await llm.exec.call(ctx, body, ctx);
   } catch (e) {
     throw new TuneError(e.message, (llm.filename || llm.fullname || llm.name), llm.row, llm.col, llm.stack, e);
   }
@@ -1789,7 +1857,6 @@ async function toolCall(payload, ctx) {
   var lastMsg;
   lastMsg = payload.messages["slice"](-1)[0];
   if (((((typeof lastMsg !== "undefined") && (lastMsg !== null) && !Number.isNaN(lastMsg) && (typeof lastMsg.tool_calls !== "undefined") && (lastMsg.tool_calls !== null) && !Number.isNaN(lastMsg.tool_calls) && (typeof lastMsg.tool_calls.length !== "undefined") && (lastMsg.tool_calls.length !== null) && !Number.isNaN(lastMsg.tool_calls.length)) ? lastMsg.tool_calls.length : (((typeof 0 !== "undefined") && (0 !== null) && !Number.isNaN(0)) ? 0 : undefined)) === 0)) return [];
-  ctx = makeContext(ctx);
   var tools;
   tools = (payload.tools || [])
     .reduce((function(memo, tool) {
@@ -1806,7 +1873,7 @@ async function toolCall(payload, ctx) {
     tool = tools[tc.name];
     if (!tool) throw new TuneError(("tool '" + tc.name + "' not defined"), "", undefined, undefined, TuneError.ctx2stack(ctx));
     try {
-      res = await tools[tc.name].exec(JSON.parse(tc.arguments), ctx);
+      res = await tools[tc.name].exec.call(ctx, JSON.parse(tc.arguments), ctx);
     } catch (e) {
       throw new TuneError(e.message, (tool.filename || tool.fullname || tool.name), tool.row, tool.col, tool.stack, e);
     }
@@ -1858,55 +1925,29 @@ TunePromise.prototype.finally = (function(onFinally) {
 });
 
 function text2run(text, ctx, opts) {
-  var msgs, stopVal, stream, ires, ierr, ifinish, resolve, reject, p;
+  var msgs, stopVal, stream, resolve, reject, p, iter;
   if (!ctx) throw Error("context not set");
-  ctx = makeContext(ctx);
   var msgs;
   var stopVal;
   var stream;
-  var ires;
-  var ierr;
-  var ifinish;
   var resolve;
   var reject;
   msgs = [];
   stopVal = (((typeof opts !== "undefined") && (opts !== null) && !Number.isNaN(opts) && (typeof opts.stop !== "undefined") && (opts.stop !== null) && !Number.isNaN(opts.stop)) ? opts.stop : (((typeof "step" !== "undefined") && ("step" !== null) && !Number.isNaN("step")) ? "step" : undefined));
   stream = (((typeof opts !== "undefined") && (opts !== null) && !Number.isNaN(opts) && (typeof opts.stream !== "undefined") && (opts.stream !== null) && !Number.isNaN(opts.stream)) ? opts.stream : (((typeof false !== "undefined") && (false !== null) && !Number.isNaN(false)) ? false : undefined));
-  ires = undefined;
-  ierr = undefined;
-  ifinish = false;
   resolve = undefined;
   reject = undefined;
   var p;
+  var iter;
   p = new Promise((function(res, rej) {
     resolve = res;
     return (reject = rej);
   }));
-  if (stream) p = new TunePromise((function(res, rej) {
-    resolve = res;
-    return (reject = (function(err) {
-      ierr = err;
-      return rej(err);
-    }));
-  }), (async function() {
-    var val;
-    await _once((function() {
-      return (!!ires || !!ierr || !!ifinish);
-    }), (function() {
-      return undefined;
-    }));
-    if (ierr) throw ierr;
-    if (ires) {
-      var val;
-      val = ires;
-      ires = undefined;
-      return val;
-    }
-    return ifinish;
-  }));
+  iter = new AsyncIter();
 
   function stop() {
     var lastMsg, _ref;
+    if ((!msgs || !msgs.length)) return;
     var lastMsg;
     lastMsg = msgs["slice"](-1)[0];
     if ((stopVal === "step")) {
@@ -1933,7 +1974,7 @@ function text2run(text, ctx, opts) {
       res = await toolCall(payload, ctx);
       if (res.length) {
         msgs = msgs.concat(res);
-        ires = {
+        iter.result = {
           value: msgs
         };
         continue;
@@ -1947,10 +1988,14 @@ function text2run(text, ctx, opts) {
         if (((((typeof res !== "undefined") && (res !== null) && !Number.isNaN(res) && (typeof res[0] !== "undefined") && (res[0] !== null) && !Number.isNaN(res[0]) && (typeof res[0].error !== "undefined") && (res[0].error !== null) && !Number.isNaN(res[0].error)) ? res[0].error : (((typeof res !== "undefined") && (res !== null) && !Number.isNaN(res) && (typeof res.error !== "undefined") && (res.error !== null) && !Number.isNaN(res.error)) ? res.error : undefined)) || (res.object === "error"))) {
           var err;
           err = new TuneError(tpl("{type: }{message}", (((typeof res !== "undefined") && (res !== null) && !Number.isNaN(res) && (typeof res[0] !== "undefined") && (res[0] !== null) && !Number.isNaN(res[0]) && (typeof res[0].error !== "undefined") && (res[0].error !== null) && !Number.isNaN(res[0].error)) ? res[0].error : (((typeof res !== "undefined") && (res !== null) && !Number.isNaN(res) && (typeof res.error !== "undefined") && (res.error !== null) && !Number.isNaN(res.error)) ? res.error : (((typeof res !== "undefined") && (res !== null) && !Number.isNaN(res)) ? res : undefined)))));
-          err.stack = TuneError.ctx2stack(ctx);
+          err._stack = TuneError.ctx2stack(ctx);
           throw err;
         }
         msgs.push((((typeof res !== "undefined") && (res !== null) && !Number.isNaN(res) && (typeof res.message !== "undefined") && (res.message !== null) && !Number.isNaN(res.message)) ? res.message : (((typeof res !== "undefined") && (res !== null) && !Number.isNaN(res) && (typeof res.choices !== "undefined") && (res.choices !== null) && !Number.isNaN(res.choices) && (typeof res.choices[0] !== "undefined") && (res.choices[0] !== null) && !Number.isNaN(res.choices[0]) && (typeof res.choices[0].message !== "undefined") && (res.choices[0].message !== null) && !Number.isNaN(res.choices[0].message)) ? res.choices[0].message : undefined)));
+        iter.result = {
+          value: msgs,
+          done: true
+        };
         continue;
       }
       var reader;
@@ -2001,7 +2046,7 @@ function text2run(text, ctx, opts) {
               if ((((typeof chunk !== "undefined") && (chunk !== null) && !Number.isNaN(chunk) && (typeof chunk.error !== "undefined") && (chunk.error !== null) && !Number.isNaN(chunk.error)) ? chunk.error : undefined)) {
                 var err;
                 err = new TuneError(JSON.stringify((((typeof chunk !== "undefined") && (chunk !== null) && !Number.isNaN(chunk) && (typeof chunk.error !== "undefined") && (chunk.error !== null) && !Number.isNaN(chunk.error)) ? chunk.error : undefined), null, "  "));
-                err.stack = TuneError.ctx2stack(ctx);
+                err._stack = TuneError.ctx2stack(ctx);
                 throw err;
               }
               if (delta.content) {
@@ -2019,35 +2064,34 @@ function text2run(text, ctx, opts) {
               role: "assistant",
               content: ""
             });
-            it = (ires = {
+            it = (iter.result = {
               value: msgs.concat(Array(it))
             });
             return it;
           })(data += (((typeof res !== "undefined") && (res !== null) && !Number.isNaN(res) && (typeof res.value !== "undefined") && (res.value !== null) && !Number.isNaN(res.value)) ? res.value : (((typeof "" !== "undefined") && ("" !== null) && !Number.isNaN("")) ? "" : undefined)));
         }
+        msgs = iter.result.value;
       }
-      if (ires) msgs = ires.value;
     }
-    ires = {
+    return (stream ? (iter.result = {
       value: msgs,
-      done: false
-    };
-    ifinish = {
       done: true
-    };
-    return resolve(msgs);
+    }) : resolve(msgs));
   }
   doit;
   doit()
     .catch((function(e) {
       var err;
-      if ((e.name === "TuneError")) return reject(e);
       var err;
-      err = new TuneError(e.message);
-      err.stack = TuneError.ctx2stack(ctx);
-      err.error = e;
-      return reject(err);
+      err = e;
+      if ((e.name !== "TuneError")) {
+        err = new TuneError(e.message);
+        err._stack = TuneError.ctx2stack(ctx);
+        err.error = e;
+      }
+      return (stream ? (iter.err = err) : reject(err));
     }));
+  if (stream) resolve(iter);
   return p;
 }
 text2run;
@@ -2106,7 +2150,7 @@ function msg2text(msg, long) {
         break;
       case "assistant":
         _ref = (function(res) {
-          if ((msg.content && ((typeof msg.content === "string") || (msg.content instanceof String)))) {
+          if ((((typeof msg !== 'undefined') && (typeof msg.content !== 'undefined')) && ((typeof msg.content === "string") || (msg.content instanceof String)))) {
             res.push(mkline("assistant", msg.content));
           } else if (Array.isArray(msg.content)) {
             res.push(msg.content
@@ -2139,10 +2183,15 @@ function msg2text(msg, long) {
           })));
           if (msg.tool_calls) res.push(msg.tool_calls
             .map((function(tc) {
-              var args, text;
+              var args, text, _ref1, _err;
               var args;
               var text;
-              args = dJSON.parse(tc.function.arguments || "{}");
+              try {
+                _ref1 = dJSON.parse(tc.function.arguments || "{}");
+              } catch (_err) {
+                _ref1 = {}
+              }
+              args = _ref1;
               text = args.text;
               delete args.text;
               args = (Object.keys(args).length ? JSON.stringify(args) : undefined);
@@ -2254,7 +2303,7 @@ function tpl(str) {
       _ref = str.replace(/{(\W*)(\w*)(\W*)}/gm, (function(_, pre, name, post) {
         return (function(res) {
           paramIndex += 1;
-          return (res ? ((pre || "") + res + (post || "")) : "");
+          return ((typeof res !== 'undefined') ? ((pre || "") + res + (post || "")) : "");
         })(params[name || paramIndex]);
       }));
     } catch (e) {
@@ -2300,6 +2349,7 @@ fs = require("fs");
 assert = require("node:assert/strict");
 util = require("util");
 exports.makeContext = makeContext;
+exports.Context = Context;
 exports.text2roles = text2roles;
 exports.roles2text = roles2text;
 exports.text2call = text2call;

package/dist/tune.mjs

@@ -20,8 +20,10 @@ const {
   envmd,
   unescape,
   escape,
+  Context,
 } = require("./tune.js");
 export {
+  Context,
   makeContext,
   text2roles,
   roles2text,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tune-sdk",
-  "version": "0.1.5",
+  "version": "0.1.8",
   "description": "tune - your everyday LLM toolkit.",
   "main": "dist/tune.js",
   "module": "dist/tune.mjs",
@@ -23,6 +23,9 @@
   },
   "license": "MIT",
   "files": [
-    "dist"
+    "dist/tune.js",
+    "dist/tune.mjs",
+    "dist/fsctx.js",
+    "bin/cli.js"
   ]
 }