promptfoo 0.24.3 → 0.25.0

package/dist/src/providers.js

@@ -39,6 +39,7 @@ const vertex_1 = require("./providers/vertex");
 const webhook_1 = require("./providers/webhook");
 const scriptCompletion_1 = require("./providers/scriptCompletion");
 const azureopenai_1 = require("./providers/azureopenai");
+const huggingface_1 = require("./providers/huggingface");
 async function loadApiProviders(providerPaths, options = {}) {
     const { basePath, env } = options;
     if (typeof providerPaths === 'string') {
@@ -155,8 +156,23 @@ async function loadApiProvider(providerPath, context = {}) {
             throw new Error(`Unknown Anthropic model type: ${modelType}. Use one of the following providers: anthropic:completion:<model name>`);
         }
     }
+    else if (providerPath?.startsWith('huggingface:')) {
+        const splits = providerPath.split(':');
+        if (splits.length < 3) {
+            throw new Error(`Invalid Huggingface provider path: ${providerPath}. Use one of the following providers: huggingface:feature-extraction:<model name>, huggingface:text-generation:<model name>`);
+        }
+        const modelName = splits.slice(2).join(':');
+        if (splits[1] === 'feature-extraction') {
+            return new huggingface_1.HuggingfaceFeatureExtractionProvider(modelName, providerOptions);
+        }
+        else if (splits[1] === 'text-generation') {
+            return new huggingface_1.HuggingfaceTextGenerationProvider(modelName, providerOptions);
+        }
+        else {
+            throw new Error(`Invalid Huggingface provider path: ${providerPath}. Use one of the following providers: huggingface:feature-extraction:<model name>, huggingface:text-generation:<model name>`);
+        }
+    }
     else if (providerPath?.startsWith('replicate:')) {
-        // Load Replicate module
         const splits = providerPath.split(':');
         const modelName = splits.slice(1).join(':');
         return new replicate_1.ReplicateProvider(modelName, providerOptions);

package/dist/src/providers.js.map

@@ -1 +1 @@
-{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AAExB,+CAI4B;AAC5B,qDAAoE;AACpE,qDAA0D;AAC1D,iDAI6B;AAC7B,2CAAoD;AACpD,6CAAkD;AAClD,+CAA6E;AAC7E,+CAAwD;AACxD,iDAAsD;AACtD,mEAAwE;AACxE,yDAIiC;AAW1B,KAAK,UAAU,gBAAgB,CACpC,aAKoB,EACpB,UAGI,EAAE;IAEN,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAClC,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;KAClE;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE;QAC9C,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,iBAAiB;gBAC3B,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;KACH;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YAClC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;gBAChC,OAAO,eAAe,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;aACrD;iBAAM,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;gBACzC,OAAO;oBACL,EAAE,EAAE,GAAG,EAAE,CAAC,mBAAmB,GAAG,EAAE;oBAClC,OAAO,EAAE,QAAQ;iBAClB,CAAC;aACH;iBAAM,IAAI,QAAQ,CAAC,EAAE,EAAE;gBACtB,iCAAiC;gBACjC,OAAO,eAAe,CAAE,QAA4B,CAAC,EAAG,EAAE;oBACxD,OAAO,EAAE,QAAQ;oBACjB,QAAQ;oBACR,GAAG;iBACJ,CAAC,CAAC;aACJ;iBAAM;gBACL,yDAAyD;gBACzD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,cAAc,GAAI,QAA+B,CAAC,EAAE,CAAC,CAAC;gBAC5D,MAAM,OAAO,GAAG,EAAE,GAAG,cAAc,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;gBACnE,OAAO,eAAe,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;aACjE;QACH,CAAC,CAAC,CACH,CAAC;KACH;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAlDD,4CAkDC;AAEM,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,UAII,EAAE;IAEN,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,eAAe,GAAG;QACtB,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG;KACJ,CAAC;IACF,IAAI,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE;QACrC,qBAAqB;QACrB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,2CAAwB,CAAC,UAAU,EAAE;YAC9C,EAAE,EAAE,QAAQ,UAAU,EAAE;YACxB,MAAM,EAAE,EAAE,QAAQ,EAAE;SACrB,CAAC,CAAC;KACJ;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QAC9C,qBAAqB;QACrB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CAAC,SAAS,IAAI,eAAe,EAAE,eAAe,CAAC,CAAC;SACxF;aAAM,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE;YAClE,OAAO,IAAI,gCAAuB,CAAC,SAAS,IAAI,wBAAwB,EAAE,eAAe,CAAC,CAAC;SAC5F;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CAAC,SAAS,IAAI,kBAAkB,EAAE,eAAe,CAAC,CAAC;SACvF;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACrE;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACjE;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,2BAA2B;QAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,+CAAiC,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;SAC/E;aAAM,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE;YAClE,OAAO,IAAI,0CAA4B,CACrC,cAAc,IAAI,wBAAwB,EAC1C,eAAe,CAChB,CAAC;SACH;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,2CAA6B,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;SAC3E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,oCAAoC,SAAS,gGAAgG,CAC9I,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,YAAY,EAAE;YAC9B,OAAO,IAAI,uCAA2B,CAAC,SAAS,IAAI,kBAAkB,EAAE,eAAe,CAAC,CAAC;SAC1F;aAAM,IAAI,uCAA2B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACtF,OAAO,IAAI,uCAA2B,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACpE;aAAM;YACL,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,yEAAyE,CACpH,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5C,OAAO,IAAI,6BAAiB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KAC1D;IAED,IAAI,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QACvC,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,OAAO,IAAI,yBAAe,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;KACzD;SAAM,IAAI,YAAY,KAAK,OAAO,IAAI,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACxE,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,qBAAa,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACtD;SAAM,IACL,YAAY,CAAC,UAAU,CAAC,oBAAoB,CAAC;QAC7C,YAAY,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAC5C;QACA,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,gCAAuB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KAChE;SAAM,IAAI,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QAC7C,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,IAAI,uBAAc,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACvD;SAAM,IAAI,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAC3C,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,uBAAgB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACzD;SAAM,IAAI,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QAC5C,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,2BAAkB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KAC3D;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QAC/C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC5D;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAClE;aAAM,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE;YAClE,OAAO,IAAI,kCAAwB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACjE;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC5D;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AA7HD,0CA6HC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,2BAA2B,EAA3B,uCAA2B;IAC3B,iBAAiB,EAAjB,6BAAiB;IACjB,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AAExB,+CAI4B;AAC5B,qDAAoE;AACpE,qDAA0D;AAC1D,iDAI6B;AAC7B,2CAAoD;AACpD,6CAAkD;AAClD,+CAA6E;AAC7E,+CAAwD;AACxD,iDAAsD;AACtD,mEAAwE;AACxE,yDAIiC;AACjC,yDAGiC;AAW1B,KAAK,UAAU,gBAAgB,CACpC,aAKoB,EACpB,UAGI,EAAE;IAEN,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAClC,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC;KAClE;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE;QAC9C,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,iBAAiB;gBAC3B,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;KACH;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YAClC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;gBAChC,OAAO,eAAe,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;aACrD;iBAAM,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;gBACzC,OAAO;oBACL,EAAE,EAAE,GAAG,EAAE,CAAC,mBAAmB,GAAG,EAAE;oBAClC,OAAO,EAAE,QAAQ;iBAClB,CAAC;aACH;iBAAM,IAAI,QAAQ,CAAC,EAAE,EAAE;gBACtB,iCAAiC;gBACjC,OAAO,eAAe,CAAE,QAA4B,CAAC,EAAG,EAAE;oBACxD,OAAO,EAAE,QAAQ;oBACjB,QAAQ;oBACR,GAAG;iBACJ,CAAC,CAAC;aACJ;iBAAM;gBACL,yDAAyD;gBACzD,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,cAAc,GAAI,QAA+B,CAAC,EAAE,CAAC,CAAC;gBAC5D,MAAM,OAAO,GAAG,EAAE,GAAG,cAAc,EAAE,EAAE,EAAE,cAAc,CAAC,EAAE,IAAI,EAAE,EAAE,CAAC;gBACnE,OAAO,eAAe,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;aACjE;QACH,CAAC,CAAC,CACH,CAAC;KACH;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAlDD,4CAkDC;AAEM,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,UAII,EAAE;IAEN,MAAM,EAAE,OAAO,GAAG,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;IAChD,MAAM,eAAe,GAAG;QACtB,EAAE,EAAE,OAAO,CAAC,EAAE;QACd,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,GAAG;KACJ,CAAC;IACF,IAAI,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE;QACrC,qBAAqB;QACrB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,2CAAwB,CAAC,UAAU,EAAE;YAC9C,EAAE,EAAE,QAAQ,UAAU,EAAE;YACxB,MAAM,EAAE,EAAE,QAAQ,EAAE;SACrB,CAAC,CAAC;KACJ;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QAC9C,qBAAqB;QACrB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CAAC,SAAS,IAAI,eAAe,EAAE,eAAe,CAAC,CAAC;SACxF;aAAM,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE;YAClE,OAAO,IAAI,gCAAuB,CAAC,SAAS,IAAI,wBAAwB,EAAE,eAAe,CAAC,CAAC;SAC5F;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CAAC,SAAS,IAAI,kBAAkB,EAAE,eAAe,CAAC,CAAC;SACvF;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACrE;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACjE;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,2BAA2B;QAC3B,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,cAAc,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,+CAAiC,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;SAC/E;aAAM,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE;YAClE,OAAO,IAAI,0CAA4B,CACrC,cAAc,IAAI,wBAAwB,EAC1C,eAAe,CAChB,CAAC;SACH;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,2CAA6B,CAAC,cAAc,EAAE,eAAe,CAAC,CAAC;SAC3E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,oCAAoC,SAAS,gGAAgG,CAC9I,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,YAAY,EAAE;YAC9B,OAAO,IAAI,uCAA2B,CAAC,SAAS,IAAI,kBAAkB,EAAE,eAAe,CAAC,CAAC;SAC1F;aAAM,IAAI,uCAA2B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACtF,OAAO,IAAI,uCAA2B,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACpE;aAAM;YACL,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,yEAAyE,CACpH,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;YACrB,MAAM,IAAI,KAAK,CACb,sCAAsC,YAAY,6HAA6H,CAChL,CAAC;SACH;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,oBAAoB,EAAE;YACtC,OAAO,IAAI,kDAAoC,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC7E;aAAM,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,iBAAiB,EAAE;YAC1C,OAAO,IAAI,+CAAiC,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC1E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,sCAAsC,YAAY,6HAA6H,CAChL,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC5C,OAAO,IAAI,6BAAiB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KAC1D;IAED,IAAI,YAAY,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;QACvC,MAAM,UAAU,GAAG,YAAY,CAAC,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,OAAO,IAAI,yBAAe,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;KACzD;SAAM,IAAI,YAAY,KAAK,OAAO,IAAI,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACxE,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,qBAAa,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACtD;SAAM,IACL,YAAY,CAAC,UAAU,CAAC,oBAAoB,CAAC;QAC7C,YAAY,CAAC,UAAU,CAAC,mBAAmB,CAAC,EAC5C;QACA,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,gCAAuB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KAChE;SAAM,IAAI,YAAY,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE;QAC7C,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7D,OAAO,IAAI,uBAAc,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACvD;SAAM,IAAI,YAAY,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE;QAC3C,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,uBAAgB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KACzD;SAAM,IAAI,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QAC5C,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,2BAAkB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;KAC3D;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QAC/C,MAAM,MAAM,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACvC,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAE5B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC5D;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAClE;aAAM,IAAI,SAAS,KAAK,WAAW,IAAI,SAAS,KAAK,YAAY,EAAE;YAClE,OAAO,IAAI,kCAAwB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SACjE;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;SAC5D;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AA5ID,0CA4IC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,2BAA2B,EAA3B,uCAA2B;IAC3B,iBAAiB,EAAjB,6BAAiB;IACjB,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}

package/dist/src/share.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../src/share.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7E,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,GAC7B,OAAO,CAAC,MAAM,CAAC,CAmBjB"}
+{"version":3,"file":"share.d.ts","sourceRoot":"","sources":["../../src/share.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,eAAe,EAAiB,aAAa,EAAE,MAAM,SAAS,CAAC;AAE7E,wBAAsB,kBAAkB,CACtC,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,GAC7B,OAAO,CAAC,MAAM,CAAC,CAqBjB"}

package/dist/src/share.js

@@ -8,7 +8,9 @@ const node_fetch_1 = __importDefault(require("node-fetch"));
 async function createShareableUrl(results, config) {
     const sharedResults = {
         data: {
-            version: 1,
+            version: 2,
+            // TODO(ian): Take date from results, if applicable.
+            createdAt: new Date().toISOString(),
             results,
             config,
         },

package/dist/src/share.js.map

@@ -1 +1 @@
-{"version":3,"file":"share.js","sourceRoot":"","sources":["../../src/share.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAIxB,KAAK,UAAU,kBAAkB,CACtC,OAAwB,EACxB,MAA8B;IAE9B,MAAM,aAAa,GAAkB;QACnC,IAAI,EAAE;YACJ,OAAO,EAAE,CAAC;YACV,OAAO;YACP,MAAM;SACP;KACF,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,gCAAgC,EAAE;QAC7D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;SACnC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;KACpC,CAAC,CAAC;IAEH,MAAM,EAAE,EAAE,EAAE,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;IACzD,OAAO,kCAAkC,EAAE,EAAE,CAAC;AAChD,CAAC;AAtBD,gDAsBC"}
+{"version":3,"file":"share.js","sourceRoot":"","sources":["../../src/share.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAIxB,KAAK,UAAU,kBAAkB,CACtC,OAAwB,EACxB,MAA8B;IAE9B,MAAM,aAAa,GAAkB;QACnC,IAAI,EAAE;YACJ,OAAO,EAAE,CAAC;YACV,oDAAoD;YACpD,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,OAAO;YACP,MAAM;SACP;KACF,CAAC;IAEF,MAAM,QAAQ,GAAG,MAAM,IAAA,oBAAK,EAAC,gCAAgC,EAAE;QAC7D,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;SACnC;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,aAAa,CAAC;KACpC,CAAC,CAAC;IAEH,MAAM,EAAE,EAAE,EAAE,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAmB,CAAC;IACzD,OAAO,kCAAkC,EAAE,EAAE,CAAC;AAChD,CAAC;AAxBD,gDAwBC"}

package/dist/src/table.d.ts

@@ -1,4 +1,5 @@
 import Table from 'cli-table3';
 import type { EvaluateSummary } from './types';
 export declare function generateTable(summary: EvaluateSummary, tableCellMaxLength?: number, maxRows?: number): Table.Table;
+export declare function wrapTable(rows: Record<string, string | number>[]): Table.Table;
 //# sourceMappingURL=table.d.ts.map

package/dist/src/table.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,wBAAgB,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,kBAAkB,SAAM,EAAE,OAAO,SAAK,eA2C7F"}
+{"version":3,"file":"table.d.ts","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAC;AAE/C,wBAAgB,aAAa,CAAC,OAAO,EAAE,eAAe,EAAE,kBAAkB,SAAM,EAAE,OAAO,SAAK,eA2C7F;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,EAAE,eAahE"}

package/dist/src/table.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.generateTable = void 0;
+exports.wrapTable = exports.generateTable = void 0;
 const cli_table3_1 = __importDefault(require("cli-table3"));
 const chalk_1 = __importDefault(require("chalk"));
 function generateTable(summary, tableCellMaxLength = 250, maxRows = 25) {
@@ -50,4 +50,19 @@ function generateTable(summary, tableCellMaxLength = 250, maxRows = 25) {
     return table;
 }
 exports.generateTable = generateTable;
+function wrapTable(rows) {
+    const maxWidth = process.stdout.columns ? process.stdout.columns - 10 : 120;
+    const head = Object.keys(rows[0]);
+    const table = new cli_table3_1.default({
+        head,
+        colWidths: Array(head.length).fill(Math.floor(maxWidth / head.length)),
+        wordWrap: true,
+        wrapOnWordBoundary: true,
+    });
+    for (const row of rows) {
+        table.push(Object.values(row));
+    }
+    return table;
+}
+exports.wrapTable = wrapTable;
 //# sourceMappingURL=table.js.map

package/dist/src/table.js.map

@@ -1 +1 @@
-{"version":3,"file":"table.js","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAC/B,kDAA0B;AAG1B,SAAgB,aAAa,CAAC,OAAwB,EAAE,kBAAkB,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC5F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrE,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;QACpE,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,KAAK;QACzB,KAAK,EAAE;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACvB;KACF,CAAC,CAAC;IACH,4DAA4D;IAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE;QACtD,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpB,IAAI,CAAC,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACjC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAC5C;gBACD,OAAO,CAAC,CAAC;YACX,CAAC,CAAC;YACF,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACpC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAClD;gBACD,IAAI,IAAI,EAAE;oBACR,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;iBACtC;qBAAM,IAAI,CAAC,IAAI,EAAE;oBAChB,sCAAsC;oBACtC,OAAO,CACL,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC;wBACpB,IAAI;6BACD,KAAK,CAAC,KAAK,CAAC;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;6BACpD,IAAI,CAAC,KAAK,CAAC,CACf,CAAC;iBACH;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CAAC;KACJ;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AA3CD,sCA2CC"}
+{"version":3,"file":"table.js","sourceRoot":"","sources":["../../src/table.ts"],"names":[],"mappings":";;;;;;AAAA,4DAA+B;AAC/B,kDAA0B;AAG1B,SAAgB,aAAa,CAAC,OAAwB,EAAE,kBAAkB,GAAG,GAAG,EAAE,OAAO,GAAG,EAAE;IAC5F,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;IAChC,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrE,SAAS,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,UAAU,CAAC,CAAC;QACpE,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,KAAK;QACzB,KAAK,EAAE;YACL,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC;SACvB;KACF,CAAC,CAAC;IACH,4DAA4D;IAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE;QACtD,KAAK,CAAC,IAAI,CAAC;YACT,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpB,IAAI,CAAC,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACjC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAC5C;gBACD,OAAO,CAAC,CAAC;YACX,CAAC,CAAC;YACF,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,EAAE,EAAE;gBAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,kBAAkB,EAAE;oBACpC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,GAAG,KAAK,CAAC;iBAClD;gBACD,IAAI,IAAI,EAAE;oBACR,OAAO,eAAK,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;iBACtC;qBAAM,IAAI,CAAC,IAAI,EAAE;oBAChB,sCAAsC;oBACtC,OAAO,CACL,eAAK,CAAC,GAAG,CAAC,SAAS,CAAC;wBACpB,IAAI;6BACD,KAAK,CAAC,KAAK,CAAC;6BACZ,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,eAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;6BACpD,IAAI,CAAC,KAAK,CAAC,CACf,CAAC;iBACH;gBACD,OAAO,IAAI,CAAC;YACd,CAAC,CAAC;SACH,CAAC,CAAC;KACJ;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AA3CD,sCA2CC;AAED,SAAgB,SAAS,CAAC,IAAuC;IAC/D,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;IAC5E,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClC,MAAM,KAAK,GAAG,IAAI,oBAAK,CAAC;QACtB,IAAI;QACJ,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC;QACtE,QAAQ,EAAE,IAAI;QACd,kBAAkB,EAAE,IAAI;KACzB,CAAC,CAAC;IACH,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;KAChC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAbD,8BAaC"}

package/dist/src/types.d.ts

@@ -101,11 +101,33 @@ export interface EvaluateOptions {
     cache?: boolean;
 }
 export interface Prompt {
+    id?: string;
     raw: string;
     display: string;
     function?: (context: {
         vars: Record<string, string | object>;
     }) => Promise<string | object>;
+    metrics?: {
+        score: number;
+        testPassCount: number;
+        testFailCount: number;
+        assertPassCount: number;
+        assertFailCount: number;
+    };
+}
+export interface PromptWithMetadata {
+    id: string;
+    prompt: Prompt;
+    recentEvalDate: Date;
+    recentEvalId: string;
+    recentEvalFilepath: string;
+    evals: {
+        id: string;
+        filePath: string;
+        datasetId: string;
+        metrics: Prompt['metrics'];
+    }[];
+    count: number;
 }
 export interface EvaluateResult {
     provider: Pick<ProviderOptions, 'id'>;
@@ -157,7 +179,7 @@ export interface GradingResult {
     componentResults?: GradingResult[];
     assertion: Assertion | null;
 }
-type BaseAssertionTypes = 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'starts-with' | 'regex' | 'is-json' | 'contains-json' | 'javascript' | 'python' | 'similar' | 'llm-rubric' | 'model-graded-closedqa' | 'model-graded-factuality' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l' | 'levenshtein';
+type BaseAssertionTypes = 'equals' | 'contains' | 'icontains' | 'contains-all' | 'contains-any' | 'icontains-all' | 'icontains-any' | 'starts-with' | 'regex' | 'is-json' | 'contains-json' | 'javascript' | 'python' | 'similar' | 'llm-rubric' | 'model-graded-closedqa' | 'model-graded-factuality' | 'webhook' | 'rouge-n' | 'rouge-s' | 'rouge-l' | 'levenshtein';
 type NotPrefixed<T extends string> = `not-${T}`;
 export type AssertionType = BaseAssertionTypes | NotPrefixed<BaseAssertionTypes>;
 export interface Assertion {
@@ -168,6 +190,21 @@ export interface Assertion {
     provider?: GradingConfig['provider'];
     rubricPrompt?: GradingConfig['rubricPrompt'];
 }
+export interface TestCasesWithMetadataPrompt {
+    prompt: Prompt;
+    id: string;
+    evalId: string;
+    evalFilepath: string;
+}
+export interface TestCasesWithMetadata {
+    id: string;
+    testCases: string | string[] | TestCase[];
+    recentEvalDate: Date;
+    recentEvalId: string;
+    recentEvalFilepath: string;
+    count: number;
+    prompts: TestCasesWithMetadataPrompt[];
+}
 export interface TestCase<Vars = Record<string, string | string[] | object>> {
     description?: string;
     vars?: Vars;
@@ -211,16 +248,25 @@ export type UnifiedConfig = TestSuiteConfig & {
     evaluateOptions: EvaluateOptions;
     commandLineOptions: Partial<CommandLineOptions>;
 };
+export interface EvalWithMetadata {
+    id: string;
+    filePath: string;
+    date: Date;
+    config: Partial<UnifiedConfig>;
+    results: EvaluateSummary;
+}
 export interface EvaluateTestSuite extends TestSuiteConfig {
     prompts: string[];
     writeLatestResults?: boolean;
 }
 export interface SharedResults {
-    data: {
-        version: number;
-        results: EvaluateSummary;
-        config: Partial<UnifiedConfig>;
-    };
+    data: ResultsFile;
+}
+export interface ResultsFile {
+    version: number;
+    createdAt: string;
+    results: EvaluateSummary;
+    config: Partial<UnifiedConfig>;
 }
 export {};
 //# sourceMappingURL=types.d.ts.map

package/dist/src/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IAGd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,UAAU,CAAC;IAChB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CACP,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;KACvC,KACE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC/B,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,eAAe,GAAG,WAAW,CAAC;IAClD,QAAQ,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,MAAM;IACrB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAA;KAAE,KAAK,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;CAC7F;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,mBAAmB,EAAE,CAAC;QAC/B,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gBAAgB,CAAC,EAAE,aAAa,EAAE,CAAC;IACnC,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;CAC7B;AAED,KAAK,kBAAkB,GACnB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,cAAc,GACd,cAAc,GACd,aAAa,GACb,OAAO,GACP,SAAS,GACT,eAAe,GACf,YAAY,GACZ,QAAQ,GACR,SAAS,GACT,YAAY,GACZ,uBAAuB,GACvB,yBAAyB,GACzB,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,aAAa,CAAC;AAElB,KAAK,WAAW,CAAC,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;AAEhD,MAAM,MAAM,aAAa,GAAG,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAGjF,MAAM,WAAW,SAAS;IAExB,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,CAAC,EACF,MAAM,GACN,MAAM,EAAE,GACR,MAAM,GACN,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;IAGjG,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAErC,YAAY,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CAC9C;AAGD,MAAM,WAAW,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC;IAEzE,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,IAAI,CAAC;IAGZ,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IAGrB,OAAO,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,aAAa,CAAC;IAGtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;IAG5B,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAGD,MAAM,WAAW,cAAc,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAE,SAAQ,QAAQ;IACtF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACxC;AAGD,MAAM,WAAW,SAAS;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,WAAW,EAAE,CAAC;IAGzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAIlB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAG7C,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGnB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAGhC,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;AAEtD,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;AAGrE,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EACL,UAAU,GACV,UAAU,EAAE,GACZ,kBAAkB,EAAE,GACpB,eAAe,EAAE,GACjB,gBAAgB,CAAC;IAGrB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG3B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC;IAGtC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAG5C,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAGlB,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG;IAC5C,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjD,CAAC;AAGF,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,eAAe,CAAC;QACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;KAChC,CAAC;CACH"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,kBAAkB;IAEjC,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,MAAM,EAAE,MAAM,CAAC;IAGf,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IAGd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,eAAe;IAC9B,EAAE,CAAC,EAAE,UAAU,CAAC;IAChB,MAAM,CAAC,EAAE,GAAG,CAAC;IACb,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,WAAW;IAC1B,EAAE,EAAE,MAAM,MAAM,CAAC;IACjB,OAAO,EAAE,CACP,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE;QACR,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;KACvC,KACE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC/B,gBAAgB,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,yBAAyB,CAAC,CAAC;CAC3E;AAED,MAAM,WAAW,UAAU;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,gBAAgB;IAC/B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,yBAAyB;IACxC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,MAAM;IACrB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAAC;CACvB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AAEhD,MAAM,WAAW,aAAa;IAC5B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,GAAG,eAAe,GAAG,WAAW,CAAC;IAClD,QAAQ,CAAC,EAAE;QACT,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,QAAQ,CAAC,EAAE,MAAM,CAAC;QAClB,gBAAgB,CAAC,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED,MAAM,WAAW,YAAY;IAC3B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,YAAY;IAC3B,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,eAAe,CAAC,EAAE,OAAO,CAAC;IAC1B,gBAAgB,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7D,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB;AAED,MAAM,WAAW,MAAM;IACrB,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,CAAC,OAAO,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAA;KAAE,KAAK,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IAC5F,OAAO,CAAC,EAAE;QACR,KAAK,EAAE,MAAM,CAAC;QACd,aAAa,EAAE,MAAM,CAAC;QACtB,aAAa,EAAE,MAAM,CAAC;QACtB,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AAGD,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,cAAc,EAAE,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE;QACL,EAAE,EAAE,MAAM,CAAC;QACX,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;KAC5B,EAAE,CAAC;IACJ,KAAK,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,IAAI,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,SAAS,EAAE,MAAM,CAAC;IAClB,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IACjC,aAAa,CAAC,EAAE,aAAa,CAAC;CAC/B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE;QACJ,OAAO,EAAE,MAAM,EAAE,CAAC;QAClB,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,CAAC;IAEF,IAAI,EAAE;QACJ,OAAO,EAAE,mBAAmB,EAAE,CAAC;QAC/B,IAAI,EAAE,MAAM,EAAE,CAAC;KAChB,EAAE,CAAC;CACL;AAED,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,QAAQ,CAAC,UAAU,CAAC,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,cAAc,EAAE,CAAC;IAC1B,KAAK,EAAE,aAAa,CAAC;IACrB,KAAK,EAAE,aAAa,CAAC;CACtB;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,OAAO,CAAC;IACd,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,gBAAgB,CAAC,EAAE,aAAa,EAAE,CAAC;IACnC,SAAS,EAAE,SAAS,GAAG,IAAI,CAAC;CAC7B;AAED,KAAK,kBAAkB,GACnB,QAAQ,GACR,UAAU,GACV,WAAW,GACX,cAAc,GACd,cAAc,GACd,eAAe,GACf,eAAe,GACf,aAAa,GACb,OAAO,GACP,SAAS,GACT,eAAe,GACf,YAAY,GACZ,QAAQ,GACR,SAAS,GACT,YAAY,GACZ,uBAAuB,GACvB,yBAAyB,GACzB,SAAS,GACT,SAAS,GACT,SAAS,GACT,SAAS,GACT,aAAa,CAAC;AAElB,KAAK,WAAW,CAAC,CAAC,SAAS,MAAM,IAAI,OAAO,CAAC,EAAE,CAAC;AAEhD,MAAM,MAAM,aAAa,GAAG,kBAAkB,GAAG,WAAW,CAAC,kBAAkB,CAAC,CAAC;AAGjF,MAAM,WAAW,SAAS;IAExB,IAAI,EAAE,aAAa,CAAC;IAGpB,KAAK,CAAC,EACF,MAAM,GACN,MAAM,EAAE,GACR,MAAM,GACN,CAAC,CAAC,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,cAAc,EAAE,SAAS,EAAE,SAAS,KAAK,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC;IAGjG,SAAS,CAAC,EAAE,MAAM,CAAC;IAGnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAGhB,QAAQ,CAAC,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC;IAErC,YAAY,CAAC,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CAC9C;AAGD,MAAM,WAAW,2BAA2B;IAC1C,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,MAAM,WAAW,qBAAqB;IACpC,EAAE,EAAE,MAAM,CAAC;IACX,SAAS,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC;IAC1C,cAAc,EAAE,IAAI,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,kBAAkB,EAAE,MAAM,CAAC;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,2BAA2B,EAAE,CAAC;CACxC;AAGD,MAAM,WAAW,QAAQ,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC;IAEzE,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,IAAI,CAAC,EAAE,IAAI,CAAC;IAGZ,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC;IAGrB,OAAO,CAAC,EAAE,YAAY,GAAG,YAAY,GAAG,aAAa,CAAC;IAGtD,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,QAAQ;IAEvB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,MAAM,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;IAG5B,KAAK,EAAE,QAAQ,EAAE,CAAC;CACnB;AAGD,MAAM,WAAW,cAAc,CAAC,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAE,SAAQ,QAAQ;IACtF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;CACxC;AAGD,MAAM,WAAW,SAAS;IAExB,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EAAE,WAAW,EAAE,CAAC;IAGzB,OAAO,EAAE,MAAM,EAAE,CAAC;IAIlB,iBAAiB,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC;IAG7C,KAAK,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGnB,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC;IAGhC,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAEhC,MAAM,MAAM,gBAAgB,GAAG,WAAW,CAAC,SAAS,CAAC,CAAC;AAEtD,MAAM,MAAM,kBAAkB,GAAG,MAAM,CAAC,UAAU,EAAE,eAAe,CAAC,CAAC;AAGrE,MAAM,WAAW,eAAe;IAE9B,WAAW,CAAC,EAAE,MAAM,CAAC;IAGrB,SAAS,EACL,UAAU,GACV,UAAU,EAAE,GACZ,kBAAkB,EAAE,GACpB,eAAe,EAAE,GACjB,gBAAgB,CAAC;IAGrB,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IAG3B,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,QAAQ,EAAE,CAAC;IAGtC,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;IAGvB,WAAW,CAAC,EAAE,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAG5C,UAAU,CAAC,EAAE,MAAM,CAAC;IAGpB,OAAO,CAAC,EAAE,OAAO,CAAC;IAGlB,GAAG,CAAC,EAAE,YAAY,CAAC;CACpB;AAED,MAAM,MAAM,aAAa,GAAG,eAAe,GAAG;IAC5C,eAAe,EAAE,eAAe,CAAC;IACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB,CAAC,CAAC;CACjD,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;IAC/B,OAAO,EAAE,eAAe,CAAC;CAC1B;AAGD,MAAM,WAAW,iBAAkB,SAAQ,eAAe;IACxD,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAED,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,WAAW,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,eAAe,CAAC;IACzB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;CAChC"}

package/dist/src/util.d.ts

@@ -1,5 +1,5 @@
 import nunjucks from 'nunjucks';
-import type { EvaluateSummary, UnifiedConfig } from './types';
+import type { EvaluateSummary, UnifiedConfig, PromptWithMetadata, TestCase, TestCasesWithMetadata, ResultsFile, EvalWithMetadata } from './types';
 export declare function resetGlobalConfig(): void;
 export declare function readGlobalConfig(): any;
 export declare function maybeRecordFirstRun(): boolean;
@@ -11,13 +11,26 @@ export declare function getLatestResultsPath(): string;
 export declare function writeLatestResults(results: EvaluateSummary, config: Partial<UnifiedConfig>): void;
 export declare function listPreviousResults(): string[];
 export declare function cleanupOldResults(remaining?: number): void;
+export declare function filenameToDate(filename: string): string;
+export declare function dateToFilename(date: Date): string;
 export declare function readResult(name: string): {
-    results: EvaluateSummary;
-    config: Partial<UnifiedConfig>;
-} | undefined;
-export declare function readLatestResults(): {
-    results: EvaluateSummary;
-    config: Partial<UnifiedConfig>;
+    id: string;
+    result: ResultsFile;
+    createdAt: Date;
 } | undefined;
+export declare function readLatestResults(): ResultsFile | undefined;
+export declare function getPromptsForTestCases(testCases: TestCase[]): PromptWithMetadata[];
+export declare function getPromptsForTestCasesHash(testCasesSha256: string): PromptWithMetadata[];
+export declare function sha256(str: string): string;
+export declare function getPrompts(): PromptWithMetadata[];
+export declare function getPromptsWithPredicate(predicate: (result: ResultsFile) => boolean): PromptWithMetadata[];
+export declare function getTestCases(): TestCasesWithMetadata[];
+export declare function getTestCasesWithPredicate(predicate: (result: ResultsFile) => boolean): TestCasesWithMetadata[];
+export declare function getPromptFromHash(hash: string): PromptWithMetadata | undefined;
+export declare function getDatasetFromHash(hash: string): TestCasesWithMetadata | undefined;
+export declare function getEvals(): EvalWithMetadata[];
+export declare function getEvalFromHash(hash: string): EvalWithMetadata | undefined;
+export declare function getEvalsWithPredicate(predicate: (result: ResultsFile) => boolean): EvalWithMetadata[];
 export declare function getNunjucksEngine(): typeof nunjucks;
+export declare function printBorder(): void;
 //# sourceMappingURL=util.d.ts.map

package/dist/src/util.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/util.ts"],"names":[],"mappings":"AAMA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAMhC,OAAO,KAAK,EAAE,eAAe,EAAuB,aAAa,EAAE,MAAM,SAAS,CAAC;AAInF,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED,wBAAgB,gBAAgB,IAAI,GAAG,CAiBtC;AAED,wBAAgB,mBAAmB,IAAI,OAAO,CAa7C;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAgB3E;AAED,wBAAgB,WAAW,CACzB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,GAC1B,IAAI,CA8BN;AAID,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,QAkC1F;AAED,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAU9C;AAED,wBAAgB,iBAAiB,CAAC,SAAS,SAAwB,QAKlE;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,GACX;IAAE,OAAO,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE,GAAG,SAAS,CAS1E;AAED,wBAAgB,iBAAiB,IAC7B;IAAE,OAAO,EAAE,eAAe,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,CAAA;CAAE,GAC5D,SAAS,CAEZ;AAED,wBAAgB,iBAAiB,oBAKhC"}
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/util.ts"],"names":[],"mappings":"AAOA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAMhC,OAAO,KAAK,EACV,eAAe,EAEf,aAAa,EACb,kBAAkB,EAClB,QAAQ,EACR,qBAAqB,EACrB,WAAW,EAEX,gBAAgB,EACjB,MAAM,SAAS,CAAC;AAIjB,wBAAgB,iBAAiB,IAAI,IAAI,CAExC;AAED,wBAAgB,gBAAgB,IAAI,GAAG,CAiBtC;AAED,wBAAgB,mBAAmB,IAAI,OAAO,CAa7C;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAgB3E;AAED,wBAAgB,WAAW,CACzB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,GAC1B,IAAI,CAgCN;AAID,wBAAgB,sBAAsB,IAAI,MAAM,CAE/C;AAED,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,QA6B1F;AAED,wBAAgB,mBAAmB,IAAI,MAAM,EAAE,CAU9C;AAED,wBAAgB,iBAAiB,CAAC,SAAS,SAAwB,QAKlE;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,UAkB9C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,IAAI,UAExC;AAED,wBAAgB,UAAU,CACxB,IAAI,EAAE,MAAM,GACX;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAgBlE;AAED,wBAAgB,iBAAiB,IAAI,WAAW,GAAG,SAAS,CAE3D;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,QAAQ,EAAE,wBAI3D;AAED,wBAAgB,0BAA0B,CAAC,eAAe,EAAE,MAAM,wBAMjE;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,MAAM,UAEjC;AAED,wBAAgB,UAAU,yBAEzB;AAED,wBAAgB,uBAAuB,CACrC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,GAC1C,kBAAkB,EAAE,CAoDtB;AAED,wBAAgB,YAAY,4BAE3B;AAED,wBAAgB,yBAAyB,CACvC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,GAC1C,qBAAqB,EAAE,CA8DzB;AAED,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,kCAQ7C;AAED,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,qCAQ9C;AAED,wBAAgB,QAAQ,uBAEvB;AAED,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,gCAQ3C;AAED,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,GAC1C,gBAAgB,EAAE,CAqBpB;AAED,wBAAgB,iBAAiB,oBAKhC;AAED,wBAAgB,WAAW,SAG1B"}

package/dist/src/util.js

@@ -26,10 +26,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getNunjucksEngine = exports.readLatestResults = exports.readResult = exports.cleanupOldResults = exports.listPreviousResults = exports.writeLatestResults = exports.getLatestResultsPath = exports.getConfigDirectoryPath = exports.writeOutput = exports.readConfig = exports.maybeReadConfig = exports.maybeRecordFirstRun = exports.readGlobalConfig = exports.resetGlobalConfig = void 0;
+exports.printBorder = exports.getNunjucksEngine = exports.getEvalsWithPredicate = exports.getEvalFromHash = exports.getEvals = exports.getDatasetFromHash = exports.getPromptFromHash = exports.getTestCasesWithPredicate = exports.getTestCases = exports.getPromptsWithPredicate = exports.getPrompts = exports.sha256 = exports.getPromptsForTestCasesHash = exports.getPromptsForTestCases = exports.readLatestResults = exports.readResult = exports.dateToFilename = exports.filenameToDate = exports.cleanupOldResults = exports.listPreviousResults = exports.writeLatestResults = exports.getLatestResultsPath = exports.getConfigDirectoryPath = exports.writeOutput = exports.readConfig = exports.maybeReadConfig = exports.maybeRecordFirstRun = exports.readGlobalConfig = exports.resetGlobalConfig = void 0;
 const fs = __importStar(require("fs"));
 const path = __importStar(require("path"));
 const os = __importStar(require("os"));
+const crypto_1 = require("crypto");
 const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
 const js_yaml_1 = __importDefault(require("js-yaml"));
 const nunjucks_1 = __importDefault(require("nunjucks"));
@@ -103,7 +104,7 @@ exports.readConfig = readConfig;
 function writeOutput(outputPath, results, config, shareableUrl) {
     const outputExtension = outputPath.split('.').pop()?.toLowerCase();
     const outputToSimpleString = (output) => `${output.pass ? '[PASS]' : '[FAIL]'} (${output.score.toFixed(2)}) ${output.text}`;
-    if (outputExtension === 'csv' || outputExtension === 'txt') {
+    if (outputExtension === 'csv') {
         const csvOutput = (0, sync_1.stringify)([
             [...results.table.head.prompts, ...results.table.head.vars],
             ...results.table.body.map((row) => [...row.outputs.map(outputToSimpleString), ...row.vars]),
@@ -113,7 +114,7 @@ function writeOutput(outputPath, results, config, shareableUrl) {
     else if (outputExtension === 'json') {
         fs.writeFileSync(outputPath, JSON.stringify({ results, config, shareableUrl }, null, 2));
     }
-    else if (outputExtension === 'yaml' || outputExtension === 'yml') {
+    else if (outputExtension === 'yaml' || outputExtension === 'yml' || outputExtension === 'txt') {
         fs.writeFileSync(outputPath, js_yaml_1.default.dump({ results, config, shareableUrl }));
     }
     else if (outputExtension === 'html') {
@@ -145,16 +146,18 @@ exports.getLatestResultsPath = getLatestResultsPath;
 function writeLatestResults(results, config) {
     const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
     // Replace hyphens with colons (Windows compatibility).
-    const timestamp = new Date().toISOString().replace(/:/g, '-');
-    const newResultsPath = path.join(resultsDirectory, `eval-${timestamp}.json`);
+    const filename = dateToFilename(new Date());
+    const newResultsPath = path.join(resultsDirectory, filename);
     const latestResultsPath = getLatestResultsPath();
     try {
         fs.mkdirSync(resultsDirectory, { recursive: true });
-        fs.writeFileSync(newResultsPath, JSON.stringify({
-            version: 1,
+        const resultsFileData = {
+            version: 2,
+            createdAt: new Date().toISOString(),
             config,
             results,
-        }, null, 2));
+        };
+        fs.writeFileSync(newResultsPath, JSON.stringify(resultsFileData, null, 2));
         // Use copy instead of symlink to avoid issues with Windows permissions.
         try {
             // Backwards compatibility: delete old symlink.
@@ -188,12 +191,39 @@ function cleanupOldResults(remaining = RESULT_HISTORY_LENGTH) {
     }
 }
 exports.cleanupOldResults = cleanupOldResults;
+function filenameToDate(filename) {
+    const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
+    // Replace hyphens with colons where necessary (Windows compatibility).
+    const dateParts = dateString.split('T');
+    const timePart = dateParts[1].replace(/-/g, ':');
+    const formattedDateString = `${dateParts[0]}T${timePart}`;
+    const date = new Date(formattedDateString);
+    return date.toLocaleDateString('en-US', {
+        year: 'numeric',
+        month: 'long',
+        day: 'numeric',
+        hour: '2-digit',
+        minute: '2-digit',
+        second: '2-digit',
+        timeZoneName: 'short',
+    });
+}
+exports.filenameToDate = filenameToDate;
+function dateToFilename(date) {
+    return `eval-${date.toISOString().replace(/:/g, '-')}.json`;
+}
+exports.dateToFilename = dateToFilename;
 function readResult(name) {
     const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
     const resultsPath = path.join(resultsDirectory, name);
     try {
-        const results = JSON.parse(fs.readFileSync(fs.realpathSync(resultsPath), 'utf-8'));
-        return results;
+        const result = JSON.parse(fs.readFileSync(fs.realpathSync(resultsPath), 'utf-8'));
+        const createdAt = new Date(filenameToDate(name));
+        return {
+            id: sha256(JSON.stringify(result.config)),
+            result,
+            createdAt,
+        };
     }
     catch (err) {
         logger_1.default.error(`Failed to read results from ${resultsPath}:\n${err}`);
@@ -204,6 +234,195 @@ function readLatestResults() {
     return JSON.parse(fs.readFileSync(getLatestResultsPath(), 'utf-8'));
 }
 exports.readLatestResults = readLatestResults;
+function getPromptsForTestCases(testCases) {
+    const testCasesJson = JSON.stringify(testCases);
+    const testCasesSha256 = sha256(testCasesJson);
+    return getPromptsForTestCasesHash(testCasesSha256);
+}
+exports.getPromptsForTestCases = getPromptsForTestCases;
+function getPromptsForTestCasesHash(testCasesSha256) {
+    return getPromptsWithPredicate((result) => {
+        const testsJson = JSON.stringify(result.config.tests);
+        const hash = sha256(testsJson);
+        return hash === testCasesSha256;
+    });
+}
+exports.getPromptsForTestCasesHash = getPromptsForTestCasesHash;
+function sha256(str) {
+    return (0, crypto_1.createHash)('sha256').update(str).digest('hex');
+}
+exports.sha256 = sha256;
+function getPrompts() {
+    return getPromptsWithPredicate(() => true);
+}
+exports.getPrompts = getPrompts;
+function getPromptsWithPredicate(predicate) {
+    const resultsFiles = listPreviousResults();
+    const groupedPrompts = {};
+    for (const filePath of resultsFiles) {
+        const file = readResult(filePath);
+        if (!file) {
+            continue;
+        }
+        const { result, createdAt } = file;
+        if (result && predicate(result)) {
+            for (const prompt of result.results.table.head.prompts) {
+                const evalId = sha256(JSON.stringify(result.config));
+                const promptId = sha256(prompt.raw);
+                const datasetId = result.config.tests ? sha256(JSON.stringify(result.config.tests)) : '-';
+                if (promptId in groupedPrompts) {
+                    groupedPrompts[promptId].recentEvalDate = new Date(Math.max(groupedPrompts[promptId].recentEvalDate.getTime(), new Date(createdAt).getTime()));
+                    groupedPrompts[promptId].count += 1;
+                    groupedPrompts[promptId].evals.push({
+                        id: evalId,
+                        filePath,
+                        datasetId,
+                        metrics: prompt.metrics,
+                    });
+                }
+                else {
+                    groupedPrompts[promptId] = {
+                        count: 1,
+                        id: promptId,
+                        prompt,
+                        recentEvalDate: new Date(createdAt),
+                        recentEvalId: evalId,
+                        recentEvalFilepath: filePath,
+                        evals: [
+                            {
+                                id: evalId,
+                                filePath,
+                                datasetId,
+                                metrics: prompt.metrics,
+                            },
+                        ],
+                    };
+                }
+            }
+        }
+    }
+    return Object.values(groupedPrompts);
+}
+exports.getPromptsWithPredicate = getPromptsWithPredicate;
+function getTestCases() {
+    return getTestCasesWithPredicate(() => true);
+}
+exports.getTestCases = getTestCases;
+function getTestCasesWithPredicate(predicate) {
+    const resultsFiles = listPreviousResults();
+    const groupedTestCases = {};
+    for (const filePath of resultsFiles) {
+        const file = readResult(filePath);
+        if (!file) {
+            continue;
+        }
+        const { result, createdAt } = file;
+        const testCases = result?.config?.tests;
+        if (testCases && predicate(result)) {
+            const evalId = sha256(JSON.stringify(result.config));
+            const datasetId = sha256(JSON.stringify(testCases));
+            if (datasetId in groupedTestCases) {
+                groupedTestCases[datasetId].recentEvalDate = new Date(Math.max(groupedTestCases[datasetId].recentEvalDate.getTime(), new Date(createdAt).getTime()));
+                groupedTestCases[datasetId].count += 1;
+                const newPrompts = result.results.table.head.prompts.map((prompt) => ({
+                    id: sha256(prompt.raw),
+                    prompt,
+                    evalId,
+                    evalFilepath: filePath,
+                }));
+                const promptsById = {};
+                for (const prompt of groupedTestCases[datasetId].prompts.concat(newPrompts)) {
+                    if (!(prompt.id in promptsById)) {
+                        promptsById[prompt.id] = prompt;
+                    }
+                }
+                groupedTestCases[datasetId].prompts = Object.values(promptsById);
+            }
+            else {
+                const newPrompts = result.results.table.head.prompts.map((prompt) => ({
+                    id: (0, crypto_1.createHash)('sha256').update(prompt.raw).digest('hex'),
+                    prompt,
+                    evalId,
+                    evalFilepath: filePath,
+                }));
+                const promptsById = {};
+                for (const prompt of newPrompts) {
+                    if (!(prompt.id in promptsById)) {
+                        promptsById[prompt.id] = prompt;
+                    }
+                }
+                groupedTestCases[datasetId] = {
+                    id: datasetId,
+                    count: 1,
+                    testCases,
+                    recentEvalDate: new Date(createdAt),
+                    recentEvalId: evalId,
+                    recentEvalFilepath: filePath,
+                    prompts: Object.values(promptsById),
+                };
+            }
+        }
+    }
+    return Object.values(groupedTestCases);
+}
+exports.getTestCasesWithPredicate = getTestCasesWithPredicate;
+function getPromptFromHash(hash) {
+    const prompts = getPrompts();
+    for (const prompt of prompts) {
+        if (prompt.id.startsWith(hash)) {
+            return prompt;
+        }
+    }
+    return undefined;
+}
+exports.getPromptFromHash = getPromptFromHash;
+function getDatasetFromHash(hash) {
+    const datasets = getTestCases();
+    for (const dataset of datasets) {
+        if (dataset.id.startsWith(hash)) {
+            return dataset;
+        }
+    }
+    return undefined;
+}
+exports.getDatasetFromHash = getDatasetFromHash;
+function getEvals() {
+    return getEvalsWithPredicate(() => true);
+}
+exports.getEvals = getEvals;
+function getEvalFromHash(hash) {
+    const evals = getEvals();
+    for (const eval_ of evals) {
+        if (eval_.id.startsWith(hash)) {
+            return eval_;
+        }
+    }
+    return undefined;
+}
+exports.getEvalFromHash = getEvalFromHash;
+function getEvalsWithPredicate(predicate) {
+    const ret = [];
+    const resultsFiles = listPreviousResults();
+    for (const filePath of resultsFiles) {
+        const file = readResult(filePath);
+        if (!file) {
+            continue;
+        }
+        const { result, createdAt } = file;
+        if (result && predicate(result)) {
+            const evalId = sha256(JSON.stringify(result.config));
+            ret.push({
+                id: evalId,
+                filePath,
+                date: createdAt,
+                config: result.config,
+                results: result.results,
+            });
+        }
+    }
+    return ret;
+}
+exports.getEvalsWithPredicate = getEvalsWithPredicate;
 function getNunjucksEngine() {
     nunjucks_1.default.configure({
         autoescape: false,
@@ -211,4 +430,9 @@ function getNunjucksEngine() {
     return nunjucks_1.default;
 }
 exports.getNunjucksEngine = getNunjucksEngine;
+function printBorder() {
+    const border = '='.repeat((process.stdout.columns || 80) - 10);
+    logger_1.default.info(border);
+}
+exports.printBorder = printBorder;
 //# sourceMappingURL=util.js.map