promptfoo 0.96.2 → 0.98.0

package/dist/src/validators/shared.d.ts

@@ -4,16 +4,19 @@ export declare const TokenUsageSchema: z.ZodObject<{
     completion: z.ZodOptional<z.ZodNumber>;
     prompt: z.ZodOptional<z.ZodNumber>;
     total: z.ZodOptional<z.ZodNumber>;
+    numRequests: z.ZodOptional<z.ZodNumber>;
 }, "strip", z.ZodTypeAny, {
     cached?: number | undefined;
     completion?: number | undefined;
     prompt?: number | undefined;
     total?: number | undefined;
+    numRequests?: number | undefined;
 }, {
     cached?: number | undefined;
     completion?: number | undefined;
     prompt?: number | undefined;
     total?: number | undefined;
+    numRequests?: number | undefined;
 }>;
 export declare const NunjucksFilterMapSchema: z.ZodRecord<z.ZodString, z.ZodFunction<z.ZodTuple<[z.ZodAny], z.ZodAny>, z.ZodString>>;
 //# sourceMappingURL=shared.d.ts.map

package/dist/src/validators/shared.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/validators/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;EAK3B,CAAC;AAEH,eAAO,MAAM,uBAAuB,wFAGnC,CAAC"}
+{"version":3,"file":"shared.d.ts","sourceRoot":"","sources":["../../../src/validators/shared.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,eAAO,MAAM,gBAAgB;;;;;;;;;;;;;;;;;;EAM3B,CAAC;AAEH,eAAO,MAAM,uBAAuB,wFAGnC,CAAC"}

package/dist/src/validators/shared.js

@@ -7,6 +7,7 @@ exports.TokenUsageSchema = zod_1.z.object({
     completion: zod_1.z.number().optional(),
     prompt: zod_1.z.number().optional(),
     total: zod_1.z.number().optional(),
+    numRequests: zod_1.z.number().optional(),
 });
 exports.NunjucksFilterMapSchema = zod_1.z.record(zod_1.z.string(), zod_1.z.function(zod_1.z.tuple([zod_1.z.any()]).rest(zod_1.z.any()), zod_1.z.string()));
 //# sourceMappingURL=shared.js.map

package/dist/src/validators/shared.js.map

@@ -1 +1 @@
-{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/validators/shared.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAEX,QAAA,gBAAgB,GAAG,OAAC,CAAC,MAAM,CAAC;IACvC,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC7B,CAAC,CAAC;AAEU,QAAA,uBAAuB,GAAG,OAAC,CAAC,MAAM,CAC7C,OAAC,CAAC,MAAM,EAAE,EACV,OAAC,CAAC,QAAQ,CAAC,OAAC,CAAC,KAAK,CAAC,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,CACzD,CAAC"}
+{"version":3,"file":"shared.js","sourceRoot":"","sources":["../../../src/validators/shared.ts"],"names":[],"mappings":";;;AAAA,6BAAwB;AAEX,QAAA,gBAAgB,GAAG,OAAC,CAAC,MAAM,CAAC;IACvC,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,UAAU,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACjC,MAAM,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,KAAK,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,WAAW,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CACnC,CAAC,CAAC;AAEU,QAAA,uBAAuB,GAAG,OAAC,CAAC,MAAM,CAC7C,OAAC,CAAC,MAAM,EAAE,EACV,OAAC,CAAC,QAAQ,CAAC,OAAC,CAAC,KAAK,CAAC,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAC,CAAC,GAAG,EAAE,CAAC,EAAE,OAAC,CAAC,MAAM,EAAE,CAAC,CACzD,CAAC"}

package/dist/test/cache.test.js

@@ -13,6 +13,7 @@ const mockedFetchResponse = (ok, response) => {
         json: () => Promise.resolve(response),
         headers: new Headers({
             'content-type': 'application/json',
+            'x-session-id': '45',
         }),
     };
 };
@@ -30,6 +31,7 @@ describe('fetchWithCache', () => {
         expect(result).toEqual({
             cached: false,
             data: response,
+            headers: { 'content-type': 'application/json', 'x-session-id': '45' },
             status: 400,
             statusText: 'Bad Request',
         });
@@ -37,11 +39,19 @@ describe('fetchWithCache', () => {
     it('should fetch data with cache enabled', async () => {
         (0, cache_1.enableCache)();
         const url = 'https://api.example.com/data';
-        const response = { data: 'test data' };
+        const response = {
+            data: 'test data',
+        };
         mockedFetch.mockResolvedValueOnce(mockedFetchResponse(true, response));
         const result = await (0, cache_1.fetchWithCache)(url, {}, 1000);
         expect(mockedFetch).toHaveBeenCalledTimes(1);
-        expect(result).toEqual({ cached: false, data: response, status: 200, statusText: 'OK' });
+        expect(result).toEqual({
+            cached: false,
+            data: response,
+            status: 200,
+            statusText: 'OK',
+            headers: { 'x-session-id': '45', 'content-type': 'application/json' },
+        });
     });
     it('should fetch data with cache enabled after previous test', async () => {
         const url = 'https://api.example.com/data';
@@ -49,7 +59,13 @@ describe('fetchWithCache', () => {
         mockedFetch.mockResolvedValueOnce(mockedFetchResponse(true, response));
         const result = await (0, cache_1.fetchWithCache)(url, {}, 1000);
         expect(mockedFetch).toHaveBeenCalledTimes(0);
-        expect(result).toEqual({ cached: true, data: response, status: 200, statusText: 'OK' });
+        expect(result).toEqual({
+            cached: true,
+            data: response,
+            status: 200,
+            statusText: 'OK',
+            headers: { 'content-type': 'application/json', 'x-session-id': '45' },
+        });
     });
     it('should only fetch data once with cache enabled', async () => {
         (0, cache_1.enableCache)();
@@ -63,8 +79,20 @@ describe('fetchWithCache', () => {
             (0, cache_1.fetchWithCache)(url, {}, 1000),
         ]);
         expect(mockedFetch).toHaveBeenCalledTimes(1);
-        expect(a).toEqual({ cached: false, data: response, status: 200, statusText: 'OK' });
-        expect(b).toEqual({ cached: true, data: response, status: 200, statusText: 'OK' });
+        expect(a).toEqual({
+            cached: false,
+            data: response,
+            status: 200,
+            statusText: 'OK',
+            headers: { 'x-session-id': '45', 'content-type': 'application/json' },
+        });
+        expect(b).toEqual({
+            cached: true,
+            data: response,
+            status: 200,
+            statusText: 'OK',
+            headers: { 'x-session-id': '45', 'content-type': 'application/json' },
+        });
     });
     it('should fetch data without cache for a single test', async () => {
         (0, cache_1.disableCache)();
@@ -73,7 +101,13 @@ describe('fetchWithCache', () => {
         mockedFetch.mockResolvedValueOnce(mockedFetchResponse(true, response));
         const result = await (0, cache_1.fetchWithCache)(url, {}, 1000);
         expect(mockedFetch).toHaveBeenCalledTimes(1);
-        expect(result).toEqual({ cached: false, data: response, status: 200, statusText: 'OK' });
+        expect(result).toEqual({
+            cached: false,
+            data: response,
+            status: 200,
+            statusText: 'OK',
+            headers: { 'content-type': 'application/json', 'x-session-id': '45' },
+        });
         (0, cache_1.enableCache)();
     });
     it('should still fetch data without cache for a single test', async () => {
@@ -83,7 +117,13 @@ describe('fetchWithCache', () => {
         mockedFetch.mockResolvedValueOnce(mockedFetchResponse(true, response));
         const result = await (0, cache_1.fetchWithCache)(url, {}, 1000);
         expect(mockedFetch).toHaveBeenCalledTimes(1);
-        expect(result).toEqual({ cached: false, data: response, status: 200, statusText: 'OK' });
+        expect(result).toEqual({
+            cached: false,
+            data: response,
+            status: 200,
+            statusText: 'OK',
+            headers: { 'content-type': 'application/json', 'x-session-id': '45' },
+        });
         (0, cache_1.enableCache)();
     });
 });

package/dist/test/cache.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"cache.test.js","sourceRoot":"","sources":["../../test/cache.test.ts"],"names":[],"mappings":";;AAAA,wCAAqF;AAErF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AAC3C,MAAM,CAAC,KAAK,GAAG,WAAW,CAAC;AAE3B,MAAM,mBAAmB,GAAG,CAAC,EAAW,EAAE,QAAgB,EAAY,EAAE;IACtE,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9C,OAAO;QACL,EAAE;QACF,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;QACtB,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa;QACrC,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;QACrC,OAAO,EAAE,IAAI,OAAO,CAAC;YACnB,cAAc,EAAE,kBAAkB;SACnC,CAAC;KACS,CAAC;AAChB,CAAC,CAAC;AAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,SAAS,CAAC,GAAG,EAAE;QACb,WAAW,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,IAAA,mBAAW,GAAE,CAAC;QAEd,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAExE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,aAAa;SAC1B,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,IAAA,mBAAW,GAAE,CAAC;QAEd,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1F,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,IAAA,mBAAW,GAAE,CAAC;QACd,IAAA,kBAAU,GAAE,CAAC;QAEb,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QACvE,WAAW,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;QAEjE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC/B,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;YAC7B,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;SAC9B,CAAC,CAAC;QAEH,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QACpF,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,IAAA,oBAAY,GAAE,CAAC;QAEf,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzF,IAAA,mBAAW,GAAE,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,IAAA,oBAAY,GAAE,CAAC;QAEf,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAEzF,IAAA,mBAAW,GAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"cache.test.js","sourceRoot":"","sources":["../../test/cache.test.ts"],"names":[],"mappings":";;AAAA,wCAAqF;AAErF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AAC3C,MAAM,CAAC,KAAK,GAAG,WAAW,CAAC;AAE3B,MAAM,mBAAmB,GAAG,CAAC,EAAW,EAAE,QAAgB,EAAY,EAAE;IACtE,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC9C,OAAO;QACL,EAAE;QACF,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;QACtB,UAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa;QACrC,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC;QACrC,OAAO,EAAE,IAAI,OAAO,CAAC;YACnB,cAAc,EAAE,kBAAkB;YAClC,cAAc,EAAE,IAAI;SACrB,CAAC;KACS,CAAC;AAChB,CAAC,CAAC;AAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,SAAS,CAAC,GAAG,EAAE;QACb,WAAW,CAAC,SAAS,EAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,IAAA,mBAAW,GAAE,CAAC;QAEd,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAC;QAExE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAE7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,cAAc,EAAE,IAAI,EAAE;YACrE,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,aAAa;SAC1B,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;QACpD,IAAA,mBAAW,GAAE,CAAC;QAEd,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG;YACf,IAAI,EAAE,WAAW;SAClB,CAAC;QAEF,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,cAAc,EAAE,kBAAkB,EAAE;SACtE,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,cAAc,EAAE,IAAI,EAAE;SACtE,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;QAC9D,IAAA,mBAAW,GAAE,CAAC;QACd,IAAA,kBAAU,GAAE,CAAC;QAEb,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QACvE,WAAW,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;QAEjE,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC/B,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;YAC7B,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC;SAC9B,CAAC,CAAC;QAEH,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAChB,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,cAAc,EAAE,kBAAkB,EAAE;SACtE,CAAC,CAAC;QACH,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC;YAChB,MAAM,EAAE,IAAI;YACZ,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,EAAE,cAAc,EAAE,IAAI,EAAE,cAAc,EAAE,kBAAkB,EAAE;SACtE,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;QACjE,IAAA,oBAAY,GAAE,CAAC;QAEf,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,cAAc,EAAE,IAAI,EAAE;SACtE,CAAC,CAAC;QAEH,IAAA,mBAAW,GAAE,CAAC;IAChB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACvE,IAAA,oBAAY,GAAE,CAAC;QAEf,MAAM,GAAG,GAAG,8BAA8B,CAAC;QAC3C,MAAM,QAAQ,GAAG,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAEvC,WAAW,CAAC,qBAAqB,CAAC,mBAAmB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;QAEvE,MAAM,MAAM,GAAG,MAAM,IAAA,sBAAc,EAAC,GAAG,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;QAEnD,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7C,MAAM,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC;YACrB,MAAM,EAAE,KAAK;YACb,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,GAAG;YACX,UAAU,EAAE,IAAI;YAChB,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,cAAc,EAAE,IAAI,EAAE;SACtE,CAAC,CAAC;QAEH,IAAA,mBAAW,GAAE,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/commands/init.test.js

@@ -30,6 +30,20 @@ const commander_1 = require("commander");
 const promises_1 = __importDefault(require("fs/promises"));
 const init = __importStar(require("../../src/commands/init"));
 const logger_1 = __importDefault(require("../../src/logger"));
+// Add mock for redteam init
+jest.mock('../../src/redteam/commands/init', () => ({
+    redteamInit: jest.fn(),
+}));
+jest.mock('../../src/server/server', () => ({
+    startServer: jest.fn(),
+    BrowserBehavior: {
+        ASK: 0,
+        OPEN: 1,
+        SKIP: 2,
+        OPEN_TO_REPORT: 3,
+        OPEN_TO_REDTEAM_CREATE: 4,
+    },
+}));
 jest.mock('../../src/commands/init', () => {
     const actual = jest.requireActual('../../src/commands/init');
     return {

package/dist/test/commands/init.test.js.map

@@ -1 +1 @@
-{"version":3,"file":"init.test.js","sourceRoot":"","sources":["../../../test/commands/init.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAAoC;AACpC,2DAA6B;AAC7B,8DAAgD;AAChD,8DAAsC;AAEtC,IAAI,CAAC,IAAI,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;IAC7D,OAAO;QACL,GAAG,MAAM;QACT,iBAAiB,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC;QACpD,eAAe,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC;QAChD,eAAe,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC;KACjD,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AACzB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;AACjC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;AAC9B,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;AAClC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;AACjC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;AAC/B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AAC7B,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;AAE9B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC;AAEzB,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,IAAI;gBACR,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,CAAC;aAClD,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,IAAI,CAAC,YAAY,CAAC,8BAA8B,EAAE,mBAAmB,CAAC,CAAC;YAE7E,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,CAAC,CAAC;YACvE,MAAM,CAAC,kBAAE,CAAC,SAAS,CAAC,CAAC,oBAAoB,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,WAAW;aACxB,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,MAAM,CACV,IAAI,CAAC,YAAY,CAAC,8BAA8B,EAAE,mBAAmB,CAAC,CACvE,CAAC,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,SAAS,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAExD,MAAM,MAAM,CACV,IAAI,CAAC,YAAY,CAAC,8BAA8B,EAAE,mBAAmB,CAAC,CACvE,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,KAAK;gBACT,UAAU,EAAE,WAAW;aACxB,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAChF,+CAA+C,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,SAAS,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAExD,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAChF,eAAe,CAChB,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,IAAI,CAAC,KAAK,CAAC,kBAAE,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;YAE1E,MAAM,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC9E,+CAA+C,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,IAAI,CAAC,KAAK,CAAC,kBAAE,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAEpF,MAAM,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC9E,2CAA2C,CAC5C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,IAAI;gBACR,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;oBAChC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE;oBACjC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE;oBACjC,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE;iBACzC,CAAC;aACH,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAE9C,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,WAAW;aACxB,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAE9C,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,gBAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,SAAS,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAExD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAE9C,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,gBAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,IAAI,OAAgB,CAAC;QAErB,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;YACxB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,CAAC;YACtE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YACvC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,CAAC;YACtE,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9B,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,IAAI,CACjC,4DAA4D,CAC7D,CAAC;YACF,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
+{"version":3,"file":"init.test.js","sourceRoot":"","sources":["../../../test/commands/init.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,yCAAoC;AACpC,2DAA6B;AAC7B,8DAAgD;AAChD,8DAAsC;AAEtC,4BAA4B;AAC5B,IAAI,CAAC,IAAI,CAAC,iCAAiC,EAAE,GAAG,EAAE,CAAC,CAAC;IAClD,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE;CACvB,CAAC,CAAC,CAAC;AAEJ,IAAI,CAAC,IAAI,CAAC,yBAAyB,EAAE,GAAG,EAAE,CAAC,CAAC;IAC1C,WAAW,EAAE,IAAI,CAAC,EAAE,EAAE;IACtB,eAAe,EAAE;QACf,GAAG,EAAE,CAAC;QACN,IAAI,EAAE,CAAC;QACP,IAAI,EAAE,CAAC;QACP,cAAc,EAAE,CAAC;QACjB,sBAAsB,EAAE,CAAC;KAC1B;CACF,CAAC,CAAC,CAAC;AAEJ,IAAI,CAAC,IAAI,CAAC,yBAAyB,EAAE,GAAG,EAAE;IACxC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,yBAAyB,CAAC,CAAC;IAC7D,OAAO;QACL,GAAG,MAAM;QACT,iBAAiB,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,iBAAiB,CAAC;QACpD,eAAe,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC;QAChD,eAAe,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC;KACjD,CAAC;AACJ,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;AACzB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AAClB,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;AACjC,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;AAC9B,IAAI,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;AAClC,IAAI,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;AACjC,IAAI,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;AAC/B,IAAI,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;AAC7B,IAAI,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;AAE9B,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,CAAC;AACzC,MAAM,CAAC,KAAK,GAAG,SAAS,CAAC;AAEzB,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;IAC5B,UAAU,CAAC,GAAG,EAAE;QACd,IAAI,CAAC,aAAa,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,GAAG,EAAE;QACb,IAAI,CAAC,eAAe,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,cAAc,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,qCAAqC,EAAE,KAAK,IAAI,EAAE;YACnD,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,IAAI;gBACR,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,cAAc,CAAC;aAClD,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,IAAI,CAAC,YAAY,CAAC,8BAA8B,EAAE,mBAAmB,CAAC,CAAC;YAE7E,MAAM,CAAC,SAAS,CAAC,CAAC,oBAAoB,CAAC,8BAA8B,CAAC,CAAC;YACvE,MAAM,CAAC,kBAAE,CAAC,SAAS,CAAC,CAAC,oBAAoB,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC;QACjF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,yCAAyC,EAAE,KAAK,IAAI,EAAE;YACvD,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,WAAW;aACxB,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,MAAM,CACV,IAAI,CAAC,YAAY,CAAC,8BAA8B,EAAE,mBAAmB,CAAC,CACvE,CAAC,OAAO,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;QAC1D,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,SAAS,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAExD,MAAM,MAAM,CACV,IAAI,CAAC,YAAY,CAAC,8BAA8B,EAAE,mBAAmB,CAAC,CACvE,CAAC,OAAO,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;YAC1E,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,KAAK;gBACT,UAAU,EAAE,WAAW;aACxB,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAChF,+CAA+C,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,SAAS,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAExD,MAAM,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAChF,eAAe,CAChB,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,mDAAmD,EAAE,KAAK,IAAI,EAAE;YACjE,IAAI,CAAC,KAAK,CAAC,kBAAE,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;YAE1E,MAAM,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC9E,+CAA+C,CAChD,CAAC;QACJ,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YAChE,IAAI,CAAC,KAAK,CAAC,kBAAE,EAAE,OAAO,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;YACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;YAEpF,MAAM,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,SAAS,EAAE,iBAAiB,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAC9E,2CAA2C,CAC5C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;QAC/B,EAAE,CAAC,kCAAkC,EAAE,KAAK,IAAI,EAAE;YAChD,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,IAAI;gBACR,MAAM,EAAE,GAAG;gBACX,IAAI,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC;oBAChC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE;oBACjC,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,KAAK,EAAE;oBACjC,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,MAAM,EAAE;iBACzC,CAAC;aACH,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAE9C,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC,CAAC;QACrD,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,gDAAgD,EAAE,KAAK,IAAI,EAAE;YAC9D,MAAM,YAAY,GAAG;gBACnB,EAAE,EAAE,KAAK;gBACT,MAAM,EAAE,GAAG;gBACX,UAAU,EAAE,WAAW;aACxB,CAAC;YACF,SAAS,CAAC,iBAAiB,CAAC,YAAY,CAAC,CAAC;YAE1C,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAE9C,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,gBAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC,CAAC;QAClF,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAC5C,SAAS,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;YAExD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC;YAE9C,MAAM,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC7B,MAAM,CAAC,gBAAM,CAAC,KAAK,CAAC,CAAC,oBAAoB,CAAC,MAAM,CAAC,gBAAgB,CAAC,eAAe,CAAC,CAAC,CAAC;QACtF,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC3B,IAAI,OAAgB,CAAC;QAErB,UAAU,CAAC,GAAG,EAAE;YACd,OAAO,GAAG,IAAI,mBAAO,EAAE,CAAC;YACxB,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;YAC1B,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,CAAC;YACtE,IAAI,CAAC,OAAO,EAAE,CAAC;gBACb,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;YACvC,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;YAClD,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,MAAM,CAAC,CAAC;YACtE,MAAM,CAAC,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9B,MAAM,CAAC,OAAO,EAAE,WAAW,EAAE,CAAC,CAAC,IAAI,CACjC,4DAA4D,CAC7D,CAAC;YACF,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QAC3C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/evaluator.test.js

@@ -25,28 +25,28 @@ const mockApiProvider = {
     id: jest.fn().mockReturnValue('test-provider'),
     callApi: jest.fn().mockResolvedValue({
         output: 'Test output',
-        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
     }),
 };
 const mockApiProvider2 = {
     id: jest.fn().mockReturnValue('test-provider-2'),
     callApi: jest.fn().mockResolvedValue({
         output: 'Test output',
-        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
     }),
 };
 const mockGradingApiProviderPasses = {
     id: jest.fn().mockReturnValue('test-grading-provider'),
     callApi: jest.fn().mockResolvedValue({
         output: JSON.stringify({ pass: true, reason: 'Test grading output' }),
-        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
     }),
 };
 const mockGradingApiProviderFails = {
     id: jest.fn().mockReturnValue('test-grading-provider'),
     callApi: jest.fn().mockResolvedValue({
         output: JSON.stringify({ pass: false, reason: 'Grading failed reason' }),
-        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+        tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
     }),
 };
 function toPrompt(text) {
@@ -78,7 +78,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 10,
+            prompt: 5,
+            completion: 5,
+            cached: 0,
+            numRequests: 1,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
         expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -99,7 +105,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 10,
+            prompt: 5,
+            completion: 5,
+            cached: 0,
+            numRequests: 1,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt 1 < 2 he said "hello world"...');
         expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -120,7 +132,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 10,
+            prompt: 5,
+            completion: 5,
+            cached: 0,
+            numRequests: 1,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
         expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1.prop1 }} {{ var2 }}');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -141,7 +159,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 10,
+            prompt: 5,
+            completion: 5,
+            cached: 0,
+            numRequests: 1,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
         expect(summary.results[0].prompt.label).toBe('test display name');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -162,7 +186,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(4);
         expect(summary.stats.successes).toBe(4);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 40, prompt: 20, completion: 20, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 40,
+            prompt: 20,
+            completion: 20,
+            cached: 0,
+            numRequests: 4,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
         expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -183,7 +213,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
         expect(summary.stats.successes).toBe(2);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 20, prompt: 10, completion: 10, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 20,
+            prompt: 10,
+            completion: 10,
+            cached: 0,
+            numRequests: 2,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt value1 value2');
         expect(summary.results[0].prompt.label).toBe('Test prompt {{ var1 }} {{ var2 }}');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -199,7 +235,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 10,
+            prompt: 5,
+            completion: 5,
+            cached: 0,
+            numRequests: 1,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt');
         expect(summary.results[0].prompt.label).toBe('Test prompt');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -215,7 +257,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(3);
         expect(summary.stats.successes).toBe(3);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 30, prompt: 15, completion: 15, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 30,
+            prompt: 15,
+            completion: 15,
+            cached: 0,
+            numRequests: 3,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt');
         expect(summary.results[0].prompt.label).toBe('Test prompt');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -423,7 +471,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider-json'),
             callApi: jest.fn().mockResolvedValue({
                 output: '{"output": "testing", "value": 123}',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {
@@ -456,7 +504,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider-transform'),
             callApi: jest.fn().mockResolvedValue({
                 output: 'Original output',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
             transform: '`Transformed: ${output}`',
         };
@@ -539,12 +587,62 @@ describe('evaluator', () => {
         }));
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(2);
     });
+    it('evaluate with context in vars transform in defaultTest', async () => {
+        const mockApiProvider = {
+            id: jest.fn().mockReturnValue('test-provider'),
+            callApi: jest.fn().mockResolvedValue({
+                output: 'Test output',
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
+            }),
+        };
+        const testSuite = {
+            providers: [mockApiProvider],
+            prompts: [toPrompt('Hello {{ name }}, your age is {{ age }}')],
+            defaultTest: {
+                options: {
+                    transformVars: `return {
+              ...vars,
+              // Test that context.uuid is available
+              id: context.uuid,
+              // Test that context.prompt is available but empty
+              hasPrompt: Boolean(context.prompt)
+            }`,
+                },
+            },
+            tests: [
+                {
+                    vars: {
+                        name: 'Alice',
+                        age: 25,
+                    },
+                },
+            ],
+        };
+        const evalRecord = await eval_1.default.create({}, testSuite.prompts, { id: (0, crypto_1.randomUUID)() });
+        await (0, evaluator_1.evaluate)(testSuite, evalRecord, {});
+        const summary = await evalRecord.toEvaluateSummary();
+        expect(summary).toEqual(expect.objectContaining({
+            stats: expect.objectContaining({
+                successes: 1,
+                failures: 0,
+            }),
+            results: expect.arrayContaining([
+                expect.objectContaining({
+                    vars: expect.objectContaining({
+                        id: expect.stringMatching(/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i),
+                        hasPrompt: true,
+                    }),
+                }),
+            ]),
+        }));
+        expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
+    });
     it('evaluate with provider transform and test transform', async () => {
         const mockApiProviderWithTransform = {
             id: jest.fn().mockReturnValue('test-provider-transform'),
             callApi: jest.fn().mockResolvedValue({
                 output: 'Original output',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
             transform: '`ProviderTransformed: ${output}`',
         };
@@ -608,7 +706,13 @@ describe('evaluator', () => {
         expect(mockApiProvider.callApi).toHaveBeenCalledTimes(1);
         expect(summary.stats.successes).toBe(1);
         expect(summary.stats.failures).toBe(0);
-        expect(summary.stats.tokenUsage).toEqual({ total: 10, prompt: 5, completion: 5, cached: 0 });
+        expect(summary.stats.tokenUsage).toEqual({
+            total: 10,
+            prompt: 5,
+            completion: 5,
+            cached: 0,
+            numRequests: 1,
+        });
         expect(summary.results[0].prompt.raw).toBe('Test prompt 1');
         expect(summary.results[0].prompt.label).toBe('Test prompt 1');
         expect(summary.results[0].response?.output).toBe('Test output');
@@ -618,7 +722,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider'),
             callApi: jest.fn().mockResolvedValue({
                 output: 'Test output',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {
@@ -656,11 +760,11 @@ describe('evaluator', () => {
                 .fn()
                 .mockResolvedValueOnce({
                 output: 'Hola mundo',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             })
                 .mockResolvedValueOnce({
                 output: 'Bonjour le monde',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {
@@ -711,19 +815,19 @@ describe('evaluator', () => {
                 .fn()
                 .mockResolvedValueOnce({
                 output: 'Spanish Hola',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             })
                 .mockResolvedValueOnce({
                 output: 'Spanish Bonjour',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             })
                 .mockResolvedValueOnce({
                 output: 'French Hola',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             })
                 .mockResolvedValueOnce({
                 output: 'French Bonjour',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {
@@ -768,7 +872,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider'),
             callApi: jest.fn().mockResolvedValue({
                 output: 'Hello, World',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {
@@ -871,7 +975,7 @@ describe('evaluator', () => {
             callApi: jest.fn().mockImplementation((prompt) => {
                 return Promise.resolve({
                     output: prompt,
-                    tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                    tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
                 });
             }),
         };
@@ -902,14 +1006,14 @@ describe('evaluator', () => {
             label: 'Labeled Provider',
             callApi: jest.fn().mockResolvedValue({
                 output: 'Labeled Provider Output',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
         };
         const mockUnlabeledProvider = {
             id: () => 'unlabeled-provider-id',
             callApi: jest.fn().mockResolvedValue({
                 output: 'Unlabeled Provider Output',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {
@@ -1029,7 +1133,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider-transform'),
             callApi: jest.fn().mockResolvedValue({
                 output: 'Original output',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
             transform: '`Provider: ${output}`',
         };
@@ -1067,7 +1171,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider-transform'),
             callApi: jest.fn().mockResolvedValue({
                 output: 'Original output',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
             transform: '`Provider: ${output}`',
         };
@@ -1112,7 +1216,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider-no-output'),
             callApi: jest.fn().mockResolvedValue({
                 output: null,
-                tokenUsage: { total: 5, prompt: 5, completion: 0, cached: 0 },
+                tokenUsage: { total: 5, prompt: 5, completion: 0, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {
@@ -1135,7 +1239,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider-no-output'),
             callApi: jest.fn().mockResolvedValue({
                 output: false,
-                tokenUsage: { total: 5, prompt: 5, completion: 0, cached: 0 },
+                tokenUsage: { total: 5, prompt: 5, completion: 0, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {
@@ -1157,7 +1261,7 @@ describe('evaluator', () => {
             id: jest.fn().mockReturnValue('test-provider'),
             callApi: jest.fn().mockResolvedValue({
                 output: 'Test response',
-                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0 },
+                tokenUsage: { total: 10, prompt: 5, completion: 5, cached: 0, numRequests: 1 },
             }),
         };
         const testSuite = {