promptfoo 0.17.9 → 0.18.0

package/dist/src/providers.js

@@ -38,11 +38,25 @@ async function loadApiProviders(providerPaths, basePath) {
     if (typeof providerPaths === 'string') {
         return [await loadApiProvider(providerPaths, undefined, basePath)];
     }
+    else if (typeof providerPaths === 'function') {
+        return [
+            {
+                id: () => 'custom-function',
+                callApi: providerPaths,
+            },
+        ];
+    }
     else if (Array.isArray(providerPaths)) {
-        return Promise.all(providerPaths.map((provider) => {
+        return Promise.all(providerPaths.map((provider, idx) => {
             if (typeof provider === 'string') {
                 return loadApiProvider(provider, undefined, basePath);
             }
+            else if (typeof provider === 'function') {
+                return {
+                    id: () => `custom-function-${idx}`,
+                    callApi: provider,
+                };
+            }
             else {
                 const id = Object.keys(provider)[0];
                 const context = { ...provider[id], id };

package/dist/src/providers.js.map

@@ -1 +1 @@
-{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AAIxB,+CAA4F;AAC5F,qDAAoE;AACpE,qDAA0D;AAC1D,iDAAqF;AACrF,mEAAwE;AACxE,yDAGiC;AAE1B,KAAK,UAAU,gBAAgB,CACpC,aAA8D,EAC9D,QAAiB;IAEjB,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;KACpE;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;YAC7B,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;gBAChC,OAAO,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;aACvD;iBAAM;gBACL,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,OAAO,GAAG,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;gBACxC,OAAO,eAAe,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;aAC/C;QACH,CAAC,CAAC,CACH,CAAC;KACH;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AApBD,4CAoBC;AAEM,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,OAAwB,EACxB,QAAiB;IAEjB,IAAI,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE;QACrC,qBAAqB;QACrB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,2CAAwB,CAAC,UAAU,EAAE;YAC9C,EAAE,EAAE,QAAQ,UAAU,EAAE;YACxB,MAAM,EAAE,EAAE,QAAQ,EAAE;SACrB,CAAC,CAAC;KACJ;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QAC9C,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CACrC,SAAS,IAAI,eAAe,EAC5B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CACjC,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAChF;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC5E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,2BAA2B;QAC3B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,+CAAiC,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC1F;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,2CAA6B,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SACtF;aAAM;YACL,MAAM,IAAI,KAAK,CACb,oCAAoC,SAAS,gGAAgG,CAC9I,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,YAAY,EAAE;YAC9B,OAAO,IAAI,uCAA2B,CACpC,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,uCAA2B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACtF,OAAO,IAAI,uCAA2B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC/E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,yEAAyE,CACpH,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE7C,OAAO,IAAI,6BAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;KACrE;IAED,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QACxC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,CAAC,CAAC;SACjD;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAlGD,0CAkGC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,2BAA2B,EAA3B,uCAA2B;IAC3B,iBAAiB,EAAjB,6BAAiB;IACjB,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AAExB,+CAA4F;AAC5F,qDAAoE;AACpE,qDAA0D;AAC1D,iDAAqF;AACrF,mEAAwE;AACxE,yDAGiC;AAU1B,KAAK,UAAU,gBAAgB,CACpC,aAAiF,EACjF,QAAiB;IAEjB,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;KACpE;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE;QAC9C,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,iBAAiB;gBAC3B,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;KACH;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YAClC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;gBAChC,OAAO,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;aACvD;iBAAM,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;gBACzC,OAAO;oBACL,EAAE,EAAE,GAAG,EAAE,CAAC,mBAAmB,GAAG,EAAE;oBAClC,OAAO,EAAE,QAAQ;iBAClB,CAAC;aACH;iBAAM;gBACL,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,OAAO,GAAG,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;gBACxC,OAAO,eAAe,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;aAC/C;QACH,CAAC,CAAC,CACH,CAAC;KACH;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAhCD,4CAgCC;AAEM,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,OAAwB,EACxB,QAAiB;IAEjB,IAAI,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE;QACrC,qBAAqB;QACrB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,2CAAwB,CAAC,UAAU,EAAE;YAC9C,EAAE,EAAE,QAAQ,UAAU,EAAE;YACxB,MAAM,EAAE,EAAE,QAAQ,EAAE;SACrB,CAAC,CAAC;KACJ;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QAC9C,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CACrC,SAAS,IAAI,eAAe,EAC5B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CACjC,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAChF;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC5E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,2BAA2B;QAC3B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,+CAAiC,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC1F;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,2CAA6B,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SACtF;aAAM;YACL,MAAM,IAAI,KAAK,CACb,oCAAoC,SAAS,gGAAgG,CAC9I,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,YAAY,EAAE;YAC9B,OAAO,IAAI,uCAA2B,CACpC,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,uCAA2B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACtF,OAAO,IAAI,uCAA2B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC/E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,yEAAyE,CACpH,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE7C,OAAO,IAAI,6BAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;KACrE;IAED,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QACxC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,CAAC,CAAC;SACjD;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAlGD,0CAkGC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,2BAA2B,EAA3B,uCAA2B;IAC3B,iBAAiB,EAAjB,6BAAiB;IACjB,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}

package/dist/src/table.js

@@ -28,11 +28,11 @@ function generateTable(summary, tableCellMaxLength = 250, maxRows = 25) {
                     text = text.slice(0, tableCellMaxLength) + '...';
                 }
                 if (pass) {
-                    return chalk_1.default.green.bold('[PASS] ') + text;
+                    return chalk_1.default.green('[PASS] ') + text;
                 }
                 else if (!pass) {
                     // color everything red up until '---'
-                    return (chalk_1.default.red.bold('[FAIL] ') +
+                    return (chalk_1.default.red('[FAIL] ') +
                         text
                             .split('---')
                             .map((c, idx) => (idx === 0 ? chalk_1.default.red.bold(c) : c))

package/dist/src/table.js.map

@@ -1 +1 @@
-{"version":3,"file":"table.js","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAC/B,kDAA0B;AAG1B,SAAgB,aAAa,CAAC,OAAwB,EAAE,kBAAkB,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC5F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrE,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;QACpE,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,KAAK;QACzB,KAAK,EAAE;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACvB;KACF,CAAC,CAAC;IACH,4DAA4D;IAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE;QACtD,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG,CAAC,IAAI;YACX,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACpC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAClD;gBACD,IAAI,IAAI,EAAE;oBACR,OAAO,eAAK,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;iBAC3C;qBAAM,IAAI,CAAC,IAAI,EAAE;oBAChB,sCAAsC;oBACtC,OAAO,CACL,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC;wBACzB,IAAI;6BACD,KAAK,CAAC,KAAK,CAAC;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;6BACpD,IAAI,CAAC,KAAK,CAAC,CACf,CAAC;iBACH;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CAAC;KACJ;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAtCD,sCAsCC"}
+{"version":3,"file":"table.js","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAC/B,kDAA0B;AAG1B,SAAgB,aAAa,CAAC,OAAwB,EAAE,kBAAkB,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC5F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrE,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;QACpE,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,KAAK;QACzB,KAAK,EAAE;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACvB;KACF,CAAC,CAAC;IACH,4DAA4D;IAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE;QACtD,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG,CAAC,IAAI;YACX,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACpC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAClD;gBACD,IAAI,IAAI,EAAE;oBACR,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;iBACtC;qBAAM,IAAI,CAAC,IAAI,EAAE;oBAChB,sCAAsC;oBACtC,OAAO,CACL,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC;wBACpB,IAAI;6BACD,KAAK,CAAC,KAAK,CAAC;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;6BACpD,IAAI,CAAC,KAAK,CAAC,CACf,CAAC;iBACH;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CAAC;KACJ;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAtCD,sCAsCC"}

package/dist/src/types.d.ts

@@ -114,12 +114,12 @@ export interface GradingResult {
     reason: string;
     tokensUsed?: TokenUsage;
 }
-type BaseAssertionTypes = 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'starts-with' | 'regex' | 'is-json' | 'contains-json' | 'javascript' | 'similar' | 'llm-rubric' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l';
+type BaseAssertionTypes = 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'starts-with' | 'regex' | 'is-json' | 'contains-json' | 'javascript' | 'python' | 'similar' | 'llm-rubric' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l';
 type NotPrefixed<T extends string> = `not-${T}`;
 export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
 export interface Assertion {
     type: AssertionType;
-    value?: string | string[];
+    value?: string | string[] | ((output: string, testCase: AtomicTestCase, assertion: Assertion) => Promise<GradingResult>);
     threshold?: number;
     weight?: number;
     provider?: ApiProvider;
@@ -127,10 +127,14 @@ export interface Assertion {
 export interface TestCase {
     description?: string;
     vars?: Record<string, string | string[] | object>;
-    loadVars?: string | string[];
     assert?: Assertion[];
     options?: PromptConfig & OutputConfig & GradingConfig;
 }
+export interface Scenario {
+    description?: string;
+    config: Partial<TestCase>[];
+    tests: TestCase[];
+}
 export interface AtomicTestCase extends TestCase {
     vars?: Record<string, string | object>;
 }
@@ -140,15 +144,18 @@ export interface TestSuite {
     prompts: Prompt[];
     providerPromptMap?: Record<string, string[]>;
     tests?: TestCase[];
+    scenarios?: Scenario[];
     defaultTest?: Partial<TestCase>;
 }
 export type ProviderId = string;
+export type ProviderFunction = (prompt: string) => Promise<ProviderResponse>;
 export type RawProviderConfig = Record<ProviderId, Omit<ProviderConfig, 'id'>>;
 export interface TestSuiteConfig {
     description?: string;
-    providers: ProviderId | ProviderId[] | RawProviderConfig[];
+    providers: ProviderId | ProviderId[] | RawProviderConfig[] | ProviderFunction;
     prompts: string | string[];
     tests: string | string[] | TestCase[];
+    scenarios?: Scenario[];
     defaultTest?: Omit<TestCase, 'description'>;
     outputPath?: string;
     sharing?: boolean;

package/dist/src/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IAGf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,mBAAmB,EAAE,CAAC;QAC/B,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,KAAK,kBAAkB,GACnB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,cAAc,GACd,cAAc,GACd,aAAa,GACb,OAAO,GACP,SAAS,GACT,eAAe,GACf,YAAY,GACZ,SAAS,GACT,YAAY,GACZ,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,CAAC;AAEd,KAAK,WAAW,CAAC,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;AAEhD,MAAM,MAAM,aAAa,GAAG,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAGjF,MAAM,WAAW,SAAS;IAExB,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG1B,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB;AAGD,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;IAGlD,QAAQ,CAAC,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG7B,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IAGrB,OAAO,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,aAAa,CAAC;CACvD;AAGD,MAAM,WAAW,cAAe,SAAQ,QAAQ;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACxC;AAGD,MAAM,WAAW,SAAS;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,WAAW,EAAE,CAAC;IAGzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAIlB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAG7C,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGnB,WAAW,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;CACjC;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC;AAG/E,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,UAAU,GAAG,UAAU,EAAE,GAAG,iBAAiB,EAAE,CAAC;IAG3D,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG3B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC;IAGtC,WAAW,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAG5C,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG;IAC5C,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjD,CAAC"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IAGf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,UAAU,CAAC;IACf,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;CACxD;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,mBAAmB,EAAE,CAAC;QAC/B,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;CACzB;AAED,KAAK,kBAAkB,GACnB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,cAAc,GACd,cAAc,GACd,aAAa,GACb,OAAO,GACP,SAAS,GACT,eAAe,GACf,YAAY,GACZ,QAAQ,GACR,SAAS,GACT,YAAY,GACZ,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,CAAC;AAEd,KAAK,WAAW,CAAC,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;AAEhD,MAAM,MAAM,aAAa,GAAG,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAGjF,MAAM,WAAW,SAAS;IAExB,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,CAAC,EACF,MAAM,GACN,MAAM,EAAE,GACR,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;IAGjG,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB;AAGD,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC;IAGlD,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IAGrB,OAAO,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,aAAa,CAAC;CACvD;AAED,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;IAG5B,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAGD,MAAM,WAAW,cAAe,SAAQ,QAAQ;IAC9C,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACxC;AAGD,MAAM,WAAW,SAAS;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,WAAW,EAAE,CAAC;IAGzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAIlB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAG7C,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGnB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;CACjC;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,gBAAgB,GAAG,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,gBAAgB,CAAC,CAAC;AAE7E,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC,CAAC;AAG/E,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,UAAU,GAAG,UAAU,EAAE,GAAG,iBAAiB,EAAE,GAAG,gBAAgB,CAAC;IAG9E,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG3B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC;IAGtC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAG5C,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG;IAC5C,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjD,CAAC"}

package/dist/src/util.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/util.ts"],"names":[],"mappings":"AAiBA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErE,OAAO,KAAK,EAEV,MAAM,EACN,eAAe,EAEf,aAAa,EACb,QAAQ,EACR,MAAM,EAEN,SAAS,EACV,MAAM,SAAS,CAAC;AAEjB,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,aAAa,EAAE,MAAM,EAAE,GACtB,SAAS,CAAC,mBAAmB,CAAC,CAqBhC;AAcD,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED,wBAAgB,gBAAgB,IAAI,GAAG,CAiBtC;AAED,wBAAgB,mBAAmB,IAAI,OAAO,CAa7C;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAgB3E;AAQD,wBAAgB,WAAW,CACzB,iBAAiB,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7D,QAAQ,GAAE,MAAW,GACpB,MAAM,EAAE,CAyEV;AAED,wBAAsB,uBAAuB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQ1E;AAED,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,EAC9B,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAiBrD;AAED,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAqB9F;AAED,wBAAsB,SAAS,CAC7B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,GAAG,SAAS,EACjD,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAyDrB;AAED,wBAAgB,WAAW,CACzB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,GAC1B,IAAI,CA8BN;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,QAAQ,CAAC,CAqBnB;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,MAAU,GAClB,OAAO,CAAC,QAAQ,CAAC,CAYnB;AAID,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,QA8B1F;AAED,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAU9C;AAED,wBAAgB,iBAAiB,CAAC,SAAS,SAAwB,QAKlE;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,GACX;IAAE,OAAO,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE,GAAG,SAAS,CAS1E;AAED,wBAAgB,iBAAiB,IAC7B;IAAE,OAAO,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE,GAC5D,SAAS,CAGZ;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAQ9D;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,CAiBxD"}
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/util.ts"],"names":[],"mappings":"AAkBA,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAErE,OAAO,KAAK,EAEV,MAAM,EACN,eAAe,EAEf,aAAa,EACb,QAAQ,EACR,MAAM,EAEN,SAAS,EACV,MAAM,SAAS,CAAC;AAEjB,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,aAAa,EAAE,MAAM,EAAE,GACtB,SAAS,CAAC,mBAAmB,CAAC,CA8BhC;AAcD,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED,wBAAgB,gBAAgB,IAAI,GAAG,CAiBtC;AAED,wBAAgB,mBAAmB,IAAI,OAAO,CAa7C;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAgB3E;AAQD,wBAAgB,WAAW,CACzB,iBAAiB,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC7D,QAAQ,GAAE,MAAW,GACpB,MAAM,EAAE,CAyEV;AAED,wBAAsB,uBAAuB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAQ1E;AAED,wBAAsB,aAAa,CACjC,WAAW,EAAE,MAAM,GAAG,MAAM,EAAE,EAC9B,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAiBrD;AAED,wBAAsB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,GAAE,MAAW,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAqB9F;AAED,wBAAsB,SAAS,CAC7B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,GAAG,SAAS,EACjD,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAyDrB;AAED,wBAAgB,WAAW,CACzB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,GAC1B,IAAI,CA8BN;AAED,wBAAgB,gBAAgB,CAC9B,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,GACd,OAAO,CAAC,QAAQ,CAAC,CAqBnB;AAED,wBAAsB,gBAAgB,CACpC,GAAG,EAAE,WAAW,EAChB,OAAO,yBAAkB,EACzB,OAAO,EAAE,MAAM,EACf,OAAO,GAAE,MAAU,GAClB,OAAO,CAAC,QAAQ,CAAC,CAYnB;AAID,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,QA8B1F;AAED,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAU9C;AAED,wBAAgB,iBAAiB,CAAC,SAAS,SAAwB,QAKlE;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,GACX;IAAE,OAAO,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE,GAAG,SAAS,CAS1E;AAED,wBAAgB,iBAAiB,IAC7B;IAAE,OAAO,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE,GAC5D,SAAS,CAGZ;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,UAQ9D;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,QAAQ,CAiBxD"}

package/dist/src/util.js

@@ -32,6 +32,7 @@ const path = __importStar(require("path"));
 const os = __importStar(require("os"));
 const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
 const node_fetch_1 = __importDefault(require("node-fetch"));
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
 const nunjucks_1 = __importDefault(require("nunjucks"));
 const glob_1 = require("glob");
@@ -50,6 +51,8 @@ function readProviderPromptMap(config, parsedPrompts) {
     for (const prompt of parsedPrompts) {
         allPrompts.push(prompt.display);
     }
+    (0, tiny_invariant_1.default)(typeof config.providers !== 'string', 'In order to use a provider-prompt map, config.providers should be an array of objects, not a string');
+    (0, tiny_invariant_1.default)(typeof config.providers !== 'function', 'In order to use a provider-prompt map, config.providers should be an array of objects, not a function');
     for (const provider of config.providers) {
         if (typeof provider === 'object') {
             const rawProvider = provider;
@@ -406,7 +409,7 @@ function writeLatestResults(results, config) {
             config,
             results,
         }, null, 2));
-        if (fs.existsSync(latestResultsPath)) {
+        if (fs.existsSync(latestResultsPath) || fs.lstatSync(latestResultsPath).isSymbolicLink()) {
             fs.unlinkSync(latestResultsPath);
         }
         fs.symlinkSync(newResultsPath, latestResultsPath);
@@ -424,7 +427,7 @@ function listPreviousResults() {
     const sortedFiles = resultsFiles.sort((a, b) => {
         const statA = fs.statSync(path.join(directory, a));
         const statB = fs.statSync(path.join(directory, b));
-        return statB.birthtime.getTime() - statA.birthtime.getTime(); // sort in descending order
+        return statA.birthtime.getTime() - statB.birthtime.getTime(); // sort in ascending order
     });
     return sortedFiles;
 }

package/dist/src/util.js.map

@@ -1 +1 @@
-{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,uCAAyB;AAEzB,iGAA6D;AAC7D,4DAA+B;AAC/B,sDAA2B;AAC3B,wDAAgC;AAChC,+BAAgC;AAChC,+BAA0C;AAC1C,yCAAmD;AACnD,6CAA+C;AAE/C,sDAA8B;AAC9B,6CAAmD;AACnD,+BAAqC;AAgBrC,SAAgB,qBAAqB,CACnC,MAA8B,EAC9B,aAAuB;IAEvB,MAAM,GAAG,GAA6B,EAAE,CAAC;IAEzC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;QACrB,OAAO,GAAG,CAAC;KACZ;IAED,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE;QAClC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KACjC;IAED,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;YAChC,MAAM,WAAW,GAAG,QAA6B,CAAC;YAClD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,OAAO,IAAI,UAAU,CAAC;SACjD;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAxBD,sDAwBC;AAED,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,SAAS,CAAC;KAClB;AACH,CAAC;AAED,IAAI,iBAAiB,GAAQ,IAAI,CAAC;AAElC,SAAgB,iBAAiB;IAC/B,iBAAiB,GAAG,IAAI,CAAC;AAC3B,CAAC;AAFD,8CAEC;AAED,SAAgB,gBAAgB;IAC9B,IAAI,CAAC,iBAAiB,EAAE;QACtB,MAAM,SAAS,GAAG,sBAAsB,EAAE,CAAC;QAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;QAE9D,IAAI,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE;YACjC,iBAAiB,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;SACzE;aAAM;YACL,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC7B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;aAC9C;YACD,iBAAiB,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACtC,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,iBAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;SAChE;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAjBD,4CAiBC;AAED,SAAgB,mBAAmB;IACjC,2BAA2B;IAC3B,IAAI;QACF,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAClB,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC;YACrB,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,gBAAgB,CAAC,EAAE,iBAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAC3F,OAAO,IAAI,CAAC;SACb;QACD,OAAO,KAAK,CAAC;KACd;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,KAAK,CAAC;KACd;AACH,CAAC;AAbD,kDAaC;AAEM,KAAK,UAAU,eAAe,CAAC,UAAkB;IACtD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QAC9B,OAAO,SAAS,CAAC;KAClB;IACD,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;AAChC,CAAC;AALD,0CAKC;AAEM,KAAK,UAAU,UAAU,CAAC,UAAkB;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;IACvC,QAAQ,GAAG,EAAE;QACX,KAAK,OAAO;YACV,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAC;QAC9C,KAAK,KAAK;YACR,OAAO,OAAO,CAAC,UAAU,CAAkB,CAAC;QAC9C,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAkB,CAAC;YAC3E,GAAG,GAAG,CAAC,MAAM,gCAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAkB,CAAC;YAC3D,OAAO,GAAG,CAAC;QACb;YACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;KACpE;AACH,CAAC;AAhBD,gCAgBC;AAED,IAAK,eAIJ;AAJD,WAAK,eAAe;IAClB,yDAAU,CAAA;IACV,uDAAS,CAAA;IACT,uDAAS,CAAA;AACX,CAAC,EAJI,eAAe,KAAf,eAAe,QAInB;AAED,SAAgB,WAAW,CACzB,iBAA6D,EAC7D,WAAmB,EAAE;IAErB,IAAI,WAAW,GAAa,EAAE,CAAC;IAC/B,IAAI,cAAc,GAAa,EAAE,CAAC;IAElC,IAAI,SAAsC,CAAC;IAC3C,IAAI,YAAgC,CAAC;IACrC,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxD,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;QACzC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACzD,WAAW,GAAG,CAAC,YAAY,CAAC,CAAC;QAC7B,qBAAqB,CAAC,GAAG,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC;QAC3D,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC;KACpC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE;QAC3C,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE;YACrD,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAClD,qBAAqB,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YACpD,OAAO,IAAA,eAAQ,EAAC,YAAY,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QACH,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC;KACnC;SAAM,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;QAChD,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACvD,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAC3C,qBAAqB,CAAC,GAAG,CAAC,YAAY,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,OAAO,YAAY,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC;KACnC;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;YACtB,sEAAsE;YACtE,MAAM,gBAAgB,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACpD,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACrD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;gBACnD,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,qBAAqB,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;gBACpD,OAAO,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;SAC7F;aAAM;YACL,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAEzD,IAAI,OAA2B,CAAC;YAChC,IAAI,SAAS,KAAK,eAAe,CAAC,KAAK,EAAE;gBACvC,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;aAC/D;iBAAM;gBACL,OAAO,GAAG,WAAW,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;gBAE9D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;gBACvC,IAAI,GAAG,KAAK,QAAQ,EAAE;oBACpB,8BAA8B;oBAC9B,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAC/E,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE;wBAC5B,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;qBACnD;oBACD,SAAS;iBACV;aACF;YACD,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;SACpD;KACF;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,KAAK,eAAe,CAAC,KAAK,EAAE;QACtE,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACtC,cAAc,GAAG,OAAO;aACrB,KAAK,CAAC,gBAAgB,CAAC;aACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;KACvD;IACD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;KACjF;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AA5ED,kCA4EC;AAEM,KAAK,UAAU,uBAAuB,CAAC,GAAW;IACvD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,MAAM,CAAC,CAAC;IACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;QAC3B,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,EAAE,CAAC,CAAC;KACvE;IACD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtC,OAAO,OAAO,CAAC;AACjB,CAAC;AARD,0DAQC;AAEM,KAAK,UAAU,aAAa,CACjC,WAA8B,EAC9B,WAAmB,EAAE;IAErB,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE;QACnC,WAAW,GAAG,CAAC,WAAW,CAAC,CAAC;KAC7B;IAED,MAAM,GAAG,GAA+C,EAAE,CAAC;IAC3D,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,IAAA,eAAQ,EAAC,YAAY,CAAC,CAAC;QAErC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;YACrB,MAAM,QAAQ,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;SAC9B;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AApBD,sCAoBC;AAEM,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,WAAmB,EAAE;IACzE,6EAA6E;IAC7E,6BAA6B;IAC7B,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAG,IAAA,YAAS,EAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAa,EAAE,CAAC;IAExB,IAAI,aAAa,KAAK,KAAK,EAAE;QAC3B,IAAI,QAAQ,CAAC,UAAU,CAAC,uCAAuC,CAAC,EAAE;YAChE,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC;YACxD,IAAI,GAAG,IAAA,YAAQ,EAAC,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SAC7C;aAAM;YACL,IAAI,GAAG,IAAA,YAAQ,EAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SAChF;KACF;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE;QACnC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC;KAC9D;SAAM,IAAI,aAAa,KAAK,MAAM,IAAI,aAAa,KAAK,KAAK,EAAE;QAC9D,IAAI,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAmB,CAAC;KAChF;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AArBD,sCAqBC;AAEM,KAAK,UAAU,SAAS,CAC7B,KAAiD,EACjD,WAAmB,EAAE;IAErB,MAAM,GAAG,GAAe,EAAE,CAAC;IAE3B,MAAM,iBAAiB,GAAG,KAAK,EAAE,aAAqB,EAAE,EAAE;QACxD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,IAAA,eAAQ,EAAC,YAAY,CAAC,CAAC;QACzC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;YAChC,MAAM,eAAe,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAe,CAAC;YACpF,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE;gBACtC,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBACrE,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAChD,QAAQ,CAAC,IAAI,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;iBACtE;aACF;YACD,GAAG,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;SAC9B;IACH,CAAC,CAAC;IAEF,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;QAC7B,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YACnD,iCAAiC;YACjC,MAAM,iBAAiB,CAAC,KAAK,CAAC,CAAC;SAChC;aAAM;YACL,kBAAkB;YAClB,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBAC3B,MAAM,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,CAAC,WAAW,GAAG,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;SACJ;KACF;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QAC/B,KAAK,MAAM,cAAc,IAAI,KAAK,EAAE;YAClC,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE;gBACtC,yBAAyB;gBACzB,MAAM,iBAAiB,CAAC,cAAc,CAAC,CAAC;aACzC;iBAAM;gBACL,+BAA+B;gBAC/B,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;aAC1B;SACF;KACF;IAED,wCAAwC;IACxC,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE;QACtB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAC9B,MAAM,IAAI,KAAK,CACb,uEAAuE,IAAI,CAAC,SAAS,CACnF,IAAI,EACJ,IAAI,EACJ,CAAC,CACF,EAAE,CACJ,CAAC;SACH;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AA5DD,8BA4DC;AAED,SAAgB,WAAW,CACzB,UAAkB,EAClB,OAAwB,EACxB,MAA8B,EAC9B,YAA2B;IAE3B,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;IAEnE,MAAM,oBAAoB,GAAG,CAAC,MAA2B,EAAE,EAAE,CAC3D,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;IAErF,IAAI,eAAe,KAAK,KAAK,IAAI,eAAe,KAAK,KAAK,EAAE;QAC1D,MAAM,SAAS,GAAG,IAAA,gBAAS,EAAC;YAC1B,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAC5F,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;KACzC;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KAC1F;SAAM,IAAI,eAAe,KAAK,MAAM,IAAI,eAAe,KAAK,KAAK,EAAE;QAClE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;KAC5E;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,IAAA,kBAAY,GAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG;YACZ,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAC5F,CAAC;QACF,MAAM,UAAU,GAAG,kBAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE;YACjD,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;KAC1C;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;KAC5E;AACH,CAAC;AAnCD,kCAmCC;AAED,SAAgB,gBAAgB,CAC9B,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe;IAEf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QAC9B,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAExB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,OAAO,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC;aAChB,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACL,CAAC;AAzBD,4CAyBC;AAEM,KAAK,UAAU,gBAAgB,CACpC,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe,EACf,UAAkB,CAAC;IAEnB,IAAI,SAAS,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE;QAChC,IAAI;YACF,OAAO,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;SACtD;QAAC,OAAO,KAAK,EAAE;YACd,SAAS,GAAG,KAAK,CAAC;YAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,sBAAsB;YAC9D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;SAC/D;KACF;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,aAAc,SAAmB,CAAC,OAAO,EAAE,CAAC,CAAC;AAC9F,CAAC;AAjBD,4CAiBC;AAED,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAEjC,SAAgB,sBAAsB;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AAC/C,CAAC;AAFD,wDAEC;AAED,SAAgB,oBAAoB;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;AACtE,CAAC;AAFD,oDAEC;AAED,SAAgB,kBAAkB,CAAC,OAAwB,EAAE,MAA8B;IACzF,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,CAAC,CAAC;IAEvE,uDAAuD;IACvD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAE9D,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,SAAS,OAAO,CAAC,CAAC;IAC7E,MAAM,iBAAiB,GAAG,oBAAoB,EAAE,CAAC;IACjD,IAAI;QACF,EAAE,CAAC,SAAS,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,EAAE,CAAC,aAAa,CACd,cAAc,EACd,IAAI,CAAC,SAAS,CACZ;YACE,OAAO,EAAE,CAAC;YACV,MAAM;YACN,OAAO;SACR,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,IAAI,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,EAAE;YACpC,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;SAClC;QACD,EAAE,CAAC,WAAW,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;QAClD,iBAAiB,EAAE,CAAC;KACrB;IAAC,OAAO,GAAG,EAAE;QACZ,gBAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,MAAM,GAAG,EAAE,CAAC,CAAC;KAC9E;AACH,CAAC;AA9BD,gDA8BC;AAED,SAAgB,mBAAmB;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7C,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QACnD,OAAO,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,2BAA2B;IAC3F,CAAC,CAAC,CAAC;IACH,OAAO,WAAW,CAAC;AACrB,CAAC;AAVD,kDAUC;AAED,SAAgB,iBAAiB,CAAC,SAAS,GAAG,qBAAqB;IACjE,MAAM,WAAW,GAAG,mBAAmB,EAAE,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;QACvD,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAC9E;AACH,CAAC;AALD,8CAKC;AAED,SAAgB,UAAU,CACxB,IAAY;IAEZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,CAAC,CAAC;IACvE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACnF,OAAO,OAAO,CAAC;KAChB;IAAC,OAAO,GAAG,EAAE;QACZ,gBAAM,CAAC,KAAK,CAAC,+BAA+B,WAAW,MAAM,GAAG,EAAE,CAAC,CAAC;KACrE;AACH,CAAC;AAXD,gCAWC;AAED,SAAgB,iBAAiB;IAG/B,MAAM,iBAAiB,GAAG,oBAAoB,EAAE,CAAC;IACjD,OAAO,UAAU,CAAC,iBAAiB,CAAC,CAAC;AACvC,CAAC;AALD,8CAKC;AAED,SAAgB,gBAAgB,CAAC,IAAc,EAAE,IAAc;IAC7D,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;KACpD;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,OAAO,UAAU,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC;AACtD,CAAC;AARD,4CAQC;AAED,SAAgB,kBAAkB,CAAC,GAAW;IAC5C,MAAM,IAAI,GAA2B,EAAE,CAAC;IACxC,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QAC9C,IAAI,GAAG,KAAK,YAAY,EAAE;YACxB,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBACvB,OAAO,CAAC,IAAI,CAAC,IAAA,gCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC;aAC1C;SACF;aAAM;YACL,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SACnB;KACF;IAED,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,OAAO;KAChB,CAAC;AACJ,CAAC;AAjBD,gDAiBC"}
+{"version":3,"file":"util.js","sourceRoot":"","sources":["../../src/util.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,2CAA6B;AAC7B,uCAAyB;AAEzB,iGAA6D;AAC7D,4DAA+B;AAC/B,oEAAuC;AACvC,sDAA2B;AAC3B,wDAAgC;AAChC,+BAAgC;AAChC,+BAA0C;AAC1C,yCAAmD;AACnD,6CAA+C;AAE/C,sDAA8B;AAC9B,6CAAmD;AACnD,+BAAqC;AAgBrC,SAAgB,qBAAqB,CACnC,MAA8B,EAC9B,aAAuB;IAEvB,MAAM,GAAG,GAA6B,EAAE,CAAC;IAEzC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;QACrB,OAAO,GAAG,CAAC;KACZ;IAED,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE;QAClC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KACjC;IAED,IAAA,wBAAS,EACP,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,EACpC,qGAAqG,CACtG,CAAC;IACF,IAAA,wBAAS,EACP,OAAO,MAAM,CAAC,SAAS,KAAK,UAAU,EACtC,uGAAuG,CACxG,CAAC;IAEF,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;QACvC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;YAChC,MAAM,WAAW,GAAG,QAA6B,CAAC;YAClD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,EAAE,CAAC,CAAC,OAAO,IAAI,UAAU,CAAC;SACjD;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAjCD,sDAiCC;AAED,MAAM,gBAAgB,GAAG,KAAK,CAAC;AAE/B,SAAS,SAAS,CAAC,IAAY;IAC7B,IAAI;QACF,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACzB;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,SAAS,CAAC;KAClB;AACH,CAAC;AAED,IAAI,iBAAiB,GAAQ,IAAI,CAAC;AAElC,SAAgB,iBAAiB;IAC/B,iBAAiB,GAAG,IAAI,CAAC;AAC3B,CAAC;AAFD,8CAEC;AAED,SAAgB,gBAAgB;IAC9B,IAAI,CAAC,iBAAiB,EAAE;QACtB,MAAM,SAAS,GAAG,sBAAsB,EAAE,CAAC;QAC3C,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC,CAAC;QAE9D,IAAI,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,EAAE;YACjC,iBAAiB,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;SACzE;aAAM;YACL,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;gBAC7B,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;aAC9C;YACD,iBAAiB,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;YACtC,EAAE,CAAC,aAAa,CAAC,cAAc,EAAE,iBAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC;SAChE;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAjBD,4CAiBC;AAED,SAAgB,mBAAmB;IACjC,2BAA2B;IAC3B,IAAI;QACF,MAAM,MAAM,GAAG,gBAAgB,EAAE,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE;YAClB,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC;YACrB,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,gBAAgB,CAAC,EAAE,iBAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAC3F,OAAO,IAAI,CAAC;SACb;QACD,OAAO,KAAK,CAAC;KACd;IAAC,OAAO,GAAG,EAAE;QACZ,OAAO,KAAK,CAAC;KACd;AACH,CAAC;AAbD,kDAaC;AAEM,KAAK,UAAU,eAAe,CAAC,UAAkB;IACtD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QAC9B,OAAO,SAAS,CAAC;KAClB;IACD,OAAO,UAAU,CAAC,UAAU,CAAC,CAAC;AAChC,CAAC;AALD,0CAKC;AAEM,KAAK,UAAU,UAAU,CAAC,UAAkB;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;IACvC,QAAQ,GAAG,EAAE;QACX,KAAK,OAAO;YACV,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACrD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAkB,CAAC;QAC9C,KAAK,KAAK;YACR,OAAO,OAAO,CAAC,UAAU,CAAkB,CAAC;QAC9C,KAAK,OAAO,CAAC;QACb,KAAK,MAAM;YACT,IAAI,GAAG,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAkB,CAAC;YAC3E,GAAG,GAAG,CAAC,MAAM,gCAAU,CAAC,WAAW,CAAC,GAAG,CAAC,CAAkB,CAAC;YAC3D,OAAO,GAAG,CAAC;QACb;YACE,MAAM,IAAI,KAAK,CAAC,0CAA0C,GAAG,EAAE,CAAC,CAAC;KACpE;AACH,CAAC;AAhBD,gCAgBC;AAED,IAAK,eAIJ;AAJD,WAAK,eAAe;IAClB,yDAAU,CAAA;IACV,uDAAS,CAAA;IACT,uDAAS,CAAA;AACX,CAAC,EAJI,eAAe,KAAf,eAAe,QAInB;AAED,SAAgB,WAAW,CACzB,iBAA6D,EAC7D,WAAmB,EAAE;IAErB,IAAI,WAAW,GAAa,EAAE,CAAC;IAC/B,IAAI,cAAc,GAAa,EAAE,CAAC;IAElC,IAAI,SAAsC,CAAC;IAC3C,IAAI,YAAgC,CAAC;IACrC,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxD,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;QACzC,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,iBAAiB,CAAC,CAAC;QACzD,WAAW,GAAG,CAAC,YAAY,CAAC,CAAC;QAC7B,qBAAqB,CAAC,GAAG,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC;QAC3D,SAAS,GAAG,eAAe,CAAC,MAAM,CAAC;KACpC;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,iBAAiB,CAAC,EAAE;QAC3C,WAAW,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE;YACrD,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;YAClD,qBAAqB,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;YACpD,OAAO,IAAA,eAAQ,EAAC,YAAY,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QACH,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC;KACnC;SAAM,IAAI,OAAO,iBAAiB,KAAK,QAAQ,EAAE;QAChD,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACvD,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAC3C,qBAAqB,CAAC,GAAG,CAAC,YAAY,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,CAAC;YAChE,OAAO,YAAY,CAAC;QACtB,CAAC,CAAC,CAAC;QACH,SAAS,GAAG,eAAe,CAAC,KAAK,CAAC;KACnC;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACrC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE;YACtB,sEAAsE;YACtE,MAAM,gBAAgB,GAAG,EAAE,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;YACpD,MAAM,YAAY,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;gBACrD,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;gBACnD,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;gBAClD,qBAAqB,CAAC,GAAG,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;gBACpD,OAAO,EAAE,CAAC,YAAY,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;YAChD,CAAC,CAAC,CAAC;YACH,cAAc,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;SAC7F;aAAM;YACL,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YAEzD,IAAI,OAA2B,CAAC;YAChC,IAAI,SAAS,KAAK,eAAe,CAAC,KAAK,EAAE;gBACvC,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,UAAU,CAAC;aAC/D;iBAAM;gBACL,OAAO,GAAG,WAAW,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC;gBAE9D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,GAAG,CAAC;gBACvC,IAAI,GAAG,KAAK,QAAQ,EAAE;oBACpB,8BAA8B;oBAC9B,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBAC/E,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE;wBAC5B,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;qBACnD;oBACD,SAAS;iBACV;aACF;YACD,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,WAAW,EAAE,OAAO,EAAE,CAAC,CAAC;SACpD;KACF;IAED,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,IAAI,SAAS,KAAK,eAAe,CAAC,KAAK,EAAE;QACtE,MAAM,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACtC,cAAc,GAAG,OAAO;aACrB,KAAK,CAAC,gBAAgB,CAAC;aACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;KACvD;IACD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;KACjF;IACD,OAAO,cAAc,CAAC;AACxB,CAAC;AA5ED,kCA4EC;AAEM,KAAK,UAAU,uBAAuB,CAAC,GAAW;IACvD,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,WAAW,EAAE,oBAAoB,CAAC,CAAC;IAC9D,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,MAAM,CAAC,CAAC;IACrC,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE;QAC3B,MAAM,IAAI,KAAK,CAAC,+CAA+C,GAAG,EAAE,CAAC,CAAC;KACvE;IACD,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtC,OAAO,OAAO,CAAC;AACjB,CAAC;AARD,0DAQC;AAEM,KAAK,UAAU,aAAa,CACjC,WAA8B,EAC9B,WAAmB,EAAE;IAErB,IAAI,OAAO,WAAW,KAAK,QAAQ,EAAE;QACnC,WAAW,GAAG,CAAC,WAAW,CAAC,CAAC;KAC7B;IAED,MAAM,GAAG,GAA+C,EAAE,CAAC;IAC3D,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE;QACpC,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACxD,MAAM,KAAK,GAAG,IAAA,eAAQ,EAAC,YAAY,CAAC,CAAC;QAErC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE;YACrB,MAAM,QAAQ,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;YACxD,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;SAC9B;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AApBD,sCAoBC;AAEM,KAAK,UAAU,aAAa,CAAC,QAAgB,EAAE,WAAmB,EAAE;IACzE,6EAA6E;IAC7E,6BAA6B;IAC7B,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;IAC1D,MAAM,aAAa,GAAG,IAAA,YAAS,EAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACvD,IAAI,IAAI,GAAa,EAAE,CAAC;IAExB,IAAI,aAAa,KAAK,KAAK,EAAE;QAC3B,IAAI,QAAQ,CAAC,UAAU,CAAC,uCAAuC,CAAC,EAAE;YAChE,MAAM,OAAO,GAAG,MAAM,uBAAuB,CAAC,QAAQ,CAAC,CAAC;YACxD,IAAI,GAAG,IAAA,YAAQ,EAAC,OAAO,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SAC7C;aAAM;YACL,IAAI,GAAG,IAAA,YAAQ,EAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;SAChF;KACF;SAAM,IAAI,aAAa,KAAK,MAAM,EAAE;QACnC,IAAI,GAAG,SAAS,CAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAC,CAAC;KAC9D;SAAM,IAAI,aAAa,KAAK,MAAM,IAAI,aAAa,KAAK,KAAK,EAAE;QAC9D,IAAI,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,gBAAgB,EAAE,OAAO,CAAC,CAAmB,CAAC;KAChF;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AArBD,sCAqBC;AAEM,KAAK,UAAU,SAAS,CAC7B,KAAiD,EACjD,WAAmB,EAAE;IAErB,MAAM,GAAG,GAAe,EAAE,CAAC;IAE3B,MAAM,iBAAiB,GAAG,KAAK,EAAE,aAAqB,EAAE,EAAE;QACxD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,IAAA,eAAQ,EAAC,YAAY,CAAC,CAAC;QACzC,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE;YAChC,MAAM,eAAe,GAAG,iBAAI,CAAC,IAAI,CAAC,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAe,CAAC;YACpF,KAAK,MAAM,QAAQ,IAAI,eAAe,EAAE;gBACtC,IAAI,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;oBACrE,MAAM,gBAAgB,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAChD,QAAQ,CAAC,IAAI,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;iBACtE;aACF;YACD,GAAG,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAC;SAC9B;IACH,CAAC,CAAC;IAEF,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE;QAC7B,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE;YACnD,iCAAiC;YACjC,MAAM,iBAAiB,CAAC,KAAK,CAAC,CAAC;SAChC;aAAM;YACL,kBAAkB;YAClB,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;YAClD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;gBAC3B,MAAM,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,CAAC;gBACrC,IAAI,CAAC,WAAW,GAAG,QAAQ,GAAG,GAAG,CAAC,EAAE,CAAC;gBACrC,OAAO,IAAI,CAAC;YACd,CAAC,CAAC,CAAC;SACJ;KACF;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;QAC/B,KAAK,MAAM,cAAc,IAAI,KAAK,EAAE;YAClC,IAAI,OAAO,cAAc,KAAK,QAAQ,EAAE;gBACtC,yBAAyB;gBACzB,MAAM,iBAAiB,CAAC,cAAc,CAAC,CAAC;aACzC;iBAAM;gBACL,+BAA+B;gBAC/B,GAAG,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;aAC1B;SACF;KACF;IAED,wCAAwC;IACxC,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE;QACtB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;YAC9B,MAAM,IAAI,KAAK,CACb,uEAAuE,IAAI,CAAC,SAAS,CACnF,IAAI,EACJ,IAAI,EACJ,CAAC,CACF,EAAE,CACJ,CAAC;SACH;KACF;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AA5DD,8BA4DC;AAED,SAAgB,WAAW,CACzB,UAAkB,EAClB,OAAwB,EACxB,MAA8B,EAC9B,YAA2B;IAE3B,MAAM,eAAe,GAAG,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,CAAC;IAEnE,MAAM,oBAAoB,GAAG,CAAC,MAA2B,EAAE,EAAE,CAC3D,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC;IAErF,IAAI,eAAe,KAAK,KAAK,IAAI,eAAe,KAAK,KAAK,EAAE;QAC1D,MAAM,SAAS,GAAG,IAAA,gBAAS,EAAC;YAC1B,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAC5F,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;KACzC;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;KAC1F;SAAM,IAAI,eAAe,KAAK,MAAM,IAAI,eAAe,KAAK,KAAK,EAAE;QAClE,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,iBAAI,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC;KAC5E;SAAM,IAAI,eAAe,KAAK,MAAM,EAAE;QACrC,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,IAAA,kBAAY,GAAE,mBAAmB,EAAE,OAAO,CAAC,CAAC;QAChF,MAAM,KAAK,GAAG;YACZ,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;YAC3D,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,oBAAoB,CAAC,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;SAC5F,CAAC;QACF,MAAM,UAAU,GAAG,kBAAQ,CAAC,YAAY,CAAC,QAAQ,EAAE;YACjD,KAAK;YACL,OAAO,EAAE,OAAO,CAAC,OAAO;SACzB,CAAC,CAAC;QACH,EAAE,CAAC,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;KAC1C;SAAM;QACL,MAAM,IAAI,KAAK,CAAC,yDAAyD,CAAC,CAAC;KAC5E;AACH,CAAC;AAnCD,kCAmCC;AAED,SAAgB,gBAAgB,CAC9B,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe;IAEf,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QAC9B,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC;QAExB,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,EAAE;YAChC,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,OAAO,KAAK,CAAC,CAAC,CAAC;QAC7D,CAAC,EAAE,OAAO,CAAC,CAAC;QAEZ,IAAA,oBAAK,EAAC,GAAG,EAAE,OAAO,CAAC;aAChB,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE;YACjB,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,OAAO,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,YAAY,CAAC,SAAS,CAAC,CAAC;YACxB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CAAC,CAAC;IACP,CAAC,CAAC,CAAC;AACL,CAAC;AAzBD,4CAyBC;AAEM,KAAK,UAAU,gBAAgB,CACpC,GAAgB,EAChB,UAAuB,EAAE,EACzB,OAAe,EACf,UAAkB,CAAC;IAEnB,IAAI,SAAS,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE,EAAE;QAChC,IAAI;YACF,OAAO,MAAM,gBAAgB,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;SACtD;QAAC,OAAO,KAAK,EAAE;YACd,SAAS,GAAG,KAAK,CAAC;YAClB,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,sBAAsB;YAC9D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;SAC/D;KACF;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,OAAO,aAAc,SAAmB,CAAC,OAAO,EAAE,CAAC,CAAC;AAC9F,CAAC;AAjBD,4CAiBC;AAED,MAAM,qBAAqB,GAAG,EAAE,CAAC;AAEjC,SAAgB,sBAAsB;IACpC,OAAO,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,YAAY,CAAC,CAAC;AAC/C,CAAC;AAFD,wDAEC;AAED,SAAgB,oBAAoB;IAClC,OAAO,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAC;AACtE,CAAC;AAFD,oDAEC;AAED,SAAgB,kBAAkB,CAAC,OAAwB,EAAE,MAA8B;IACzF,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,CAAC,CAAC;IAEvE,uDAAuD;IACvD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAE9D,MAAM,cAAc,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,QAAQ,SAAS,OAAO,CAAC,CAAC;IAC7E,MAAM,iBAAiB,GAAG,oBAAoB,EAAE,CAAC;IACjD,IAAI;QACF,EAAE,CAAC,SAAS,CAAC,gBAAgB,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,EAAE,CAAC,aAAa,CACd,cAAc,EACd,IAAI,CAAC,SAAS,CACZ;YACE,OAAO,EAAE,CAAC;YACV,MAAM;YACN,OAAO;SACR,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;QACF,IAAI,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC,cAAc,EAAE,EAAE;YACxF,EAAE,CAAC,UAAU,CAAC,iBAAiB,CAAC,CAAC;SAClC;QACD,EAAE,CAAC,WAAW,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;QAClD,iBAAiB,EAAE,CAAC;KACrB;IAAC,OAAO,GAAG,EAAE;QACZ,gBAAM,CAAC,KAAK,CAAC,qCAAqC,cAAc,MAAM,GAAG,EAAE,CAAC,CAAC;KAC9E;AACH,CAAC;AA9BD,gDA8BC;AAED,SAAgB,mBAAmB;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,CAAC,CAAC;IAChE,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;IACxC,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IAChG,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7C,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QACnD,OAAO,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,0BAA0B;IAC1F,CAAC,CAAC,CAAC;IACH,OAAO,WAAW,CAAC;AACrB,CAAC;AAVD,kDAUC;AAED,SAAgB,iBAAiB,CAAC,SAAS,GAAG,qBAAqB;IACjE,MAAM,WAAW,GAAG,mBAAmB,EAAE,CAAC;IAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE;QACvD,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;KAC9E;AACH,CAAC;AALD,8CAKC;AAED,SAAgB,UAAU,CACxB,IAAY;IAEZ,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE,EAAE,QAAQ,CAAC,CAAC;IACvE,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;IACtD,IAAI;QACF,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACnF,OAAO,OAAO,CAAC;KAChB;IAAC,OAAO,GAAG,EAAE;QACZ,gBAAM,CAAC,KAAK,CAAC,+BAA+B,WAAW,MAAM,GAAG,EAAE,CAAC,CAAC;KACrE;AACH,CAAC;AAXD,gCAWC;AAED,SAAgB,iBAAiB;IAG/B,MAAM,iBAAiB,GAAG,oBAAoB,EAAE,CAAC;IACjD,OAAO,UAAU,CAAC,iBAAiB,CAAC,CAAC;AACvC,CAAC;AALD,8CAKC;AAED,SAAgB,gBAAgB,CAAC,IAAc,EAAE,IAAc;IAC7D,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;KACpD;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,GAAG,GAAG,GAAG,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/E,OAAO,UAAU,GAAG,CAAC,aAAa,GAAG,aAAa,CAAC,CAAC;AACtD,CAAC;AARD,4CAQC;AAED,SAAgB,kBAAkB,CAAC,GAAW;IAC5C,MAAM,IAAI,GAA2B,EAAE,CAAC;IACxC,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE;QAC9C,IAAI,GAAG,KAAK,YAAY,EAAE;YACxB,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;gBACvB,OAAO,CAAC,IAAI,CAAC,IAAA,gCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC;aAC1C;SACF;aAAM;YACL,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SACnB;KACF;IAED,OAAO;QACL,IAAI;QACJ,MAAM,EAAE,OAAO;KAChB,CAAC;AACJ,CAAC;AAjBD,gDAiBC"}

package/package.json

@@ -2,7 +2,7 @@
   "name": "promptfoo",
   "description": "LLM eval & testing toolkit",
   "author": "Ian Webster",
-  "version": "0.17.9",
+  "version": "0.18.0",
   "license": "MIT",
   "type": "commonjs",
   "main": "dist/src/index.js",

package/src/assertions.ts

@@ -99,12 +99,21 @@ export async function runAssertion(
     type: baseType,
   });
 
+  //render assertion values
+  let renderedValue = assertion.value;
+  // renderString for assertion values
+  if (renderedValue && typeof renderedValue === 'string') {
+    renderedValue = nunjucks.renderString(renderedValue, test.vars || {});
+  } else if (renderedValue && Array.isArray(renderedValue)) {
+    renderedValue = renderedValue.map((v) => nunjucks.renderString(v, test.vars || {}));
+  }
+
   if (baseType === 'equals') {
-    pass = assertion.value === output;
+    pass = renderedValue === output;
     return {
       pass,
       score: pass ? 1 : 0,
-      reason: pass ? 'Assertion passed' : `Expected output "${assertion.value}"`,
+      reason: pass ? 'Assertion passed' : `Expected output "${renderedValue}"`,
     };
   }
 
@@ -123,103 +132,99 @@ export async function runAssertion(
   }
 
   if (baseType === 'contains') {
-    invariant(assertion.value, '"contains" assertion type must have a string or number value');
+    invariant(renderedValue, '"contains" assertion type must have a string or number value');
     invariant(
-      typeof assertion.value === 'string' || typeof assertion.value === 'number',
+      typeof renderedValue === 'string' || typeof renderedValue === 'number',
       '"contains" assertion type must have a string or number value',
     );
-    pass = output.includes(String(assertion.value)) !== inverse;
+    pass = output.includes(String(renderedValue)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain "${assertion.value}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
     };
   }
 
   if (baseType === 'contains-any') {
-    invariant(assertion.value, '"contains-any" assertion type must have a value');
+    invariant(renderedValue, '"contains-any" assertion type must have a value');
     invariant(
-      Array.isArray(assertion.value),
+      Array.isArray(renderedValue),
       '"contains-any" assertion type must have an array value',
     );
-    pass = assertion.value.some((value) => output.includes(value)) !== inverse;
+    pass = renderedValue.some((value) => output.includes(value)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain one of "${assertion.value.join(
-            ', ',
-          )}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}contain one of "${renderedValue.join(', ')}"`,
     };
   }
 
   if (baseType === 'contains-all') {
-    invariant(assertion.value, '"contains-all" assertion type must have a value');
+    invariant(renderedValue, '"contains-all" assertion type must have a value');
     invariant(
-      Array.isArray(assertion.value),
+      Array.isArray(renderedValue),
       '"contains-all" assertion type must have an array value',
     );
-    pass = assertion.value.every((value) => output.includes(value)) !== inverse;
+    pass = renderedValue.every((value) => output.includes(value)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain all of "${assertion.value.join(
-            ', ',
-          )}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}contain all of "${renderedValue.join(', ')}"`,
     };
   }
 
   if (baseType === 'regex') {
-    invariant(assertion.value, '"regex" assertion type must have a string value');
+    invariant(renderedValue, '"regex" assertion type must have a string value');
     invariant(
-      typeof assertion.value === 'string',
+      typeof renderedValue === 'string',
       '"contains" assertion type must have a string value',
     );
-    const regex = new RegExp(assertion.value);
+    const regex = new RegExp(renderedValue);
     pass = regex.test(output) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}match regex "${assertion.value}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}match regex "${renderedValue}"`,
     };
   }
 
   if (baseType === 'icontains') {
-    invariant(assertion.value, '"icontains" assertion type must have a string or number value');
+    invariant(renderedValue, '"icontains" assertion type must have a string or number value');
     invariant(
-      typeof assertion.value === 'string' || typeof assertion.value === 'number',
+      typeof renderedValue === 'string' || typeof renderedValue === 'number',
       '"icontains" assertion type must have a string or number value',
     );
-    pass = output.toLowerCase().includes(String(assertion.value).toLowerCase()) !== inverse;
+    pass = output.toLowerCase().includes(String(renderedValue).toLowerCase()) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}contain "${assertion.value}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}contain "${renderedValue}"`,
     };
   }
 
   if (baseType === 'starts-with') {
-    invariant(assertion.value, '"starts-with" assertion type must have a string value');
+    invariant(renderedValue, '"starts-with" assertion type must have a string value');
     invariant(
-      typeof assertion.value === 'string',
+      typeof renderedValue === 'string',
       '"starts-with" assertion type must have a string value',
     );
-    pass = output.startsWith(String(assertion.value)) !== inverse;
+    pass = output.startsWith(String(renderedValue)) !== inverse;
     return {
       pass,
       score: pass ? 1 : 0,
       reason: pass
         ? 'Assertion passed'
-        : `Expected output to ${inverse ? 'not ' : ''}start with "${assertion.value}"`,
+        : `Expected output to ${inverse ? 'not ' : ''}start with "${renderedValue}"`,
     };
   }
 
@@ -234,12 +239,16 @@ export async function runAssertion(
     };
   }
 
+  const context = {
+    vars: test.vars || {},
+  };
+
   if (baseType === 'javascript') {
     try {
-      const customFunction = new Function('output', 'context', `return ${assertion.value}`);
-      const context = {
-        vars: test.vars || {},
-      };
+      if (typeof assertion.value === 'function') {
+        return assertion.value(output, test, assertion);
+      }
+      const customFunction = new Function('output', 'context', `return ${renderedValue}`);
       const result = customFunction(output, context) as any;
       if (typeof result === 'boolean') {
         pass = result !== inverse;
@@ -255,7 +264,7 @@ export async function runAssertion(
         pass: false,
         score: 0,
         reason: `Custom function threw error: ${(err as Error).message}
-${assertion.value}`,
+${renderedValue}`,
       };
     }
     return {
@@ -264,41 +273,82 @@ ${assertion.value}`,
       reason: pass
         ? 'Assertion passed'
         : `Custom function returned ${inverse ? 'true' : 'false'}
+${renderedValue}`,
+    };
+  }
+
+  if (baseType === 'python') {
+    try {
+      const { execSync } = require('child_process');
+      const escapedOutput = output.replace(/'/g, "\\'").replace(/"/g, '\\"');
+      const escapedContext = JSON.stringify(context).replace(/'/g, "\\'").replace(/"/g, '\\"');
+      const result = execSync(
+        `python -c "import json; import math; import os; import sys; import re; import datetime; import random; import collections; output='${escapedOutput}'; context='${escapedContext}'; print(json.dumps(${assertion.value}))"`,
+      )
+        .toString()
+        .trim();
+      if (result === 'true') {
+        pass = true;
+        score = 1.0;
+      } else if (result === 'false') {
+        pass = false;
+        score = 0.0;
+      } else if (result.startsWith('{')) {
+        return JSON.parse(result);
+      } else {
+        pass = true;
+        score = parseFloat(result);
+        if (isNaN(score)) {
+          throw new Error(
+            'Python code must return a boolean, number, or {pass, score, reason} object',
+          );
+        }
+      }
+    } catch (err) {
+      return {
+        pass: false,
+        score: 0,
+        reason: `Python code execution failed: ${(err as Error).message}`,
+      };
+    }
+    return {
+      pass,
+      score,
+      reason: pass
+        ? 'Assertion passed'
+        : `Python code returned ${pass ? 'true' : 'false'}
 ${assertion.value}`,
     };
   }
 
   if (baseType === 'similar') {
-    invariant(assertion.value, 'Similarity assertion must have a string value');
+    invariant(renderedValue, 'Similarity assertion must have a string value');
     invariant(
-      typeof assertion.value === 'string',
+      typeof renderedValue === 'string',
       '"contains" assertion type must have a string value',
     );
-    return matchesSimilarity(assertion.value, output, assertion.threshold || 0.75, inverse);
+    return matchesSimilarity(renderedValue, output, assertion.threshold || 0.75, inverse);
   }
 
   if (baseType === 'llm-rubric') {
-    invariant(assertion.value, 'Similarity assertion must have a string value');
+    invariant(renderedValue, 'Similarity assertion must have a string value');
     invariant(
-      typeof assertion.value === 'string',
+      typeof renderedValue === 'string',
       '"contains" assertion type must have a string value',
     );
-    return matchesLlmRubric(assertion.value, output, test.options);
+    return matchesLlmRubric(renderedValue, output, test.options);
   }
 
   if (baseType === 'webhook') {
-    invariant(assertion.value, '"webhook" assertion type must have a URL value');
-    invariant(
-      typeof assertion.value === 'string',
-      '"webhook" assertion type must have a URL value',
-    );
+    invariant(renderedValue, '"webhook" assertion type must have a URL value');
+    invariant(typeof renderedValue === 'string', '"webhook" assertion type must have a URL value');
 
     try {
       const context = {
         vars: test.vars || {},
       };
       const response = await fetchWithRetries(
-        assertion.value,
+        renderedValue,
         {
           method: 'POST',
           headers: {
@@ -339,8 +389,11 @@ ${assertion.value}`,
   }
 
   if (baseType === 'rouge-n') {
-    invariant(assertion.value, '"rouge" assertion type must a value (string or string array)');
-    return handleRougeScore(baseType, assertion, assertion.value, output, inverse);
+    invariant(
+      typeof renderedValue === 'string' || Array.isArray(renderedValue),
+      '"rouge" assertion type must be a value (string or string array)',
+    );
+    return handleRougeScore(baseType, assertion, renderedValue, output, inverse);
   }
 
   throw new Error('Unknown assertion type: ' + assertion.type);

package/src/evaluator.ts

@@ -255,10 +255,11 @@ class Evaluator {
     }
 
     // Aggregate all vars across test cases
-
-    const tests = (
+    let tests = (
       testSuite.tests && testSuite.tests.length > 0
         ? testSuite.tests
+        : testSuite.scenarios
+        ? []
         : [
             {
               // Dummy test for cases when we're only comparing raw prompts.
@@ -269,6 +270,35 @@ class Evaluator {
       return Object.assign(finalTestCase, test);
     });
 
+    // Build scenarios and add to tests
+    if (testSuite.scenarios && testSuite.scenarios.length > 0) {
+      for (const scenario of testSuite.scenarios) {
+        for (const data of scenario.config) {
+          // Merge defaultTest with scenario config
+          const scenarioTests = (
+            scenario.tests || [
+              {
+                // Dummy test for cases when we're only comparing raw prompts.
+              },
+            ]
+          ).map((test) => {
+            return {
+              ...testSuite.defaultTest,
+              ...data,
+              ...test,
+              vars: {
+                ...testSuite.defaultTest?.vars,
+                ...data.vars,
+                ...test.vars,
+              },
+            };
+          });
+          // Add scenario tests to tests
+          tests = tests.concat(scenarioTests);
+        }
+      }
+    }
+
     const varNames: Set<string> = new Set();
     const varsWithSpecialColsRemoved: Record<string, string | string[] | object>[] = [];
     for (const testCase of tests) {
@@ -352,8 +382,7 @@ class Evaluator {
     // Set up progress bar...
     let progressbar: SingleBar | undefined;
     if (options.showProgressBar) {
-      const totalNumRuns =
-        testSuite.prompts.length * testSuite.providers.length * (totalVarCombinations || 1);
+      const totalNumRuns = runEvalOptions.length;
       const cliProgress = await import('cli-progress');
       progressbar = new cliProgress.SingleBar(
         {