promptfoo 0.91.2 → 0.92.0

package/dist/src/util/index.d.ts

@@ -1,25 +1,19 @@
-import { type EvalWithMetadata, type EvaluateResult, type EvaluateSummary, type EvaluateTable, type NunjucksFilterMap, type PromptWithMetadata, type ResultsFile, type TestCase, type TestCasesWithMetadata, type UnifiedConfig, type OutputFile, type CompletedPrompt, type ResultLightweight } from '../types';
-/**
- * Checks if a file is a JavaScript or TypeScript file based on its extension.
- *
- * @param filePath - The path of the file to check.
- * @returns True if the file has a JavaScript or TypeScript extension, false otherwise.
- */
-export declare function isJavascriptFile(filePath: string): boolean;
-export declare function writeOutput(outputPath: string, evalId: string | null, results: EvaluateSummary, config: Partial<UnifiedConfig>, shareableUrl: string | null): Promise<void>;
-export declare function writeMultipleOutputs(outputPaths: string[], evalId: string | null, results: EvaluateSummary, config: Partial<UnifiedConfig>, shareableUrl: string | null): Promise<void>;
+import Eval from '../models/eval';
+import { type EvalWithMetadata, type EvaluateResult, type EvaluateTable, type NunjucksFilterMap, type PromptWithMetadata, type ResultsFile, type TestCase, type TestCasesWithMetadata, type UnifiedConfig, type OutputFile, type CompletedPrompt, type ResultLightweight, type EvaluateSummaryV2 } from '../types';
+export declare function writeOutput(outputPath: string, evalRecord: Eval, shareableUrl: string | null): Promise<void>;
+export declare function writeMultipleOutputs(outputPaths: string[], evalRecord: Eval, shareableUrl: string | null): Promise<void>;
 export declare function readOutput(outputPath: string): Promise<OutputFile>;
 /**
  * TODO(ian): Remove this
  * @deprecated Use readLatestResults directly instead.
  */
 export declare function getLatestResultsPath(): string;
-export declare function writeResultsToDatabase(results: EvaluateSummary, config: Partial<UnifiedConfig>, createdAt?: Date): Promise<string>;
+export declare function writeResultsToDatabase(results: EvaluateSummaryV2, config: Partial<UnifiedConfig>, createdAt?: Date): Promise<string>;
 /**
  *
  * @returns Last n evals in descending order.
  */
-export declare function listPreviousResults(limit?: number, filterDescription?: string, datasetId?: string): ResultLightweight[];
+export declare function listPreviousResults(limit?: number, filterDescription?: string, datasetId?: string): Promise<ResultLightweight[]>;
 /**
  * @deprecated Used only for migration to sqlite
  */
@@ -42,7 +36,6 @@ export declare function readResult_fileSystem(name: string): {
     createdAt: Date;
 } | undefined;
 export declare function migrateResultsFromFileSystemToDatabase(): Promise<void>;
-export declare function cleanupOldFileResults(remaining?: number): void;
 export declare function readResult(id: string): Promise<{
     id: string;
     result: ResultsFile;
@@ -170,6 +163,7 @@ export declare function getTestCases(limit?: number): Promise<{
         } | {
             callApi: import("../types").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -334,6 +328,7 @@ export declare function getDatasetFromHash(hash: string): Promise<{
         } | {
             callApi: import("../types").CallApiFunction;
             id: (...args: unknown[]) => string;
+            config?: any;
             label?: string | undefined;
             transform?: string | undefined;
             delay?: number | undefined;
@@ -412,7 +407,7 @@ export declare function getStandaloneEvals({ limit, tag, description, }?: {
         value: string;
     };
     description?: string;
-}): StandaloneEval[];
+}): Promise<StandaloneEval[]>;
 export declare function providerToIdentifier(provider: TestCase['provider']): string | undefined;
 export declare function varsMatch(vars1: Record<string, string | string[] | object> | undefined, vars2: Record<string, string | string[] | object> | undefined): boolean;
 export declare function resultIsForTestCase(result: EvaluateResult, testCase: TestCase): boolean;

package/dist/src/util/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/util/index.ts"],"names":[],"mappings":"AA+BA,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,aAAa,EAElB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,qBAAqB,EAE1B,KAAK,aAAa,EAClB,KAAK,UAAU,EACf,KAAK,eAAe,EAEpB,KAAK,iBAAiB,EAIvB,MAAM,UAAU,CAAC;AAOlB;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAE1D;AAuBD,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAgE5B;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,MAAM,EAAE,EACrB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAK5B;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CASxE;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,SAAS,CAAC,EAAE,IAAI,GACf,OAAO,CAAC,MAAM,CAAC,CA4HjB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,GAAE,MAA4B,EACnC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,iBAAiB,EAAE,CAmCrB;AAED;;GAEG;AACH,wBAAgB,sCAAsC,IAAI,MAAM,EAAE,CAYjE;AAID;;GAEG;AACH,wBAAgB,8BAA8B,IAAI;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,CAqB7F;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,QAqB9C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,IAAI,UAExC;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,GACX;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAgBlE;AAID,wBAAsB,sCAAsC,kBA0D3D;AAID,wBAAgB,qBAAqB,CAAC,SAAS,SAAwB,QAKtE;AAED,wBAAsB,UAAU,CAC9B,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAAC,CAsC3E;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,MAAM,EACV,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,EAClC,QAAQ,CAAC,EAAE,aAAa,GACvB,OAAO,CAAC,IAAI,CAAC,CAyCf;AAED,wBAAsB,aAAa,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAgChG;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAkE/B;AAED,wBAAgB,0BAA0B,CACxC,eAAe,EAAE,MAAM,EACvB,KAAK,GAAE,MAA4B,iCAOpC;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,QAAQ,EAAE,iCAI3D;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAwElC;AAED,wBAAgB,UAAU,CAAC,KAAK,GAAE,MAA4B,iCAE7D;AAED,wBAAsB,YAAY,CAAC,KAAK,GAAE,MAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAErE;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,2CAQnD;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAQpD;AAED,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAwC7B;AAED,wBAAsB,QAAQ,CAAC,KAAK,GAAE,MAA4B,+BAEjE;AAED,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,yCAQ/C;AAED,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,iBAa9C;AAED;;;;GAIG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAQpD;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAa5B;AAED,wBAAgB,WAAW,SAG1B;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,QAOnD;AAED,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAElB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACzC,CAAC;AAIF,wBAAgB,kBAAkB,CAAC,EACjC,KAA2B,EAC3B,GAAG,EACH,WAAW,GACZ,GAAE;IACD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,cAAc,EAAE,CA+ExB;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,SAAS,CASvF;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,EAC7D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,WAG9D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAMvF;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAsBvF;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,GACjB;IACD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAoCA;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,IAAI,OA+BhG"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/util/index.ts"],"names":[],"mappings":"AAgCA,OAAO,IAA+C,MAAM,gBAAgB,CAAC;AAE7E,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,aAAa,EAElB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,qBAAqB,EAE1B,KAAK,aAAa,EAClB,KAAK,UAAU,EACf,KAAK,eAAe,EAEpB,KAAK,iBAAiB,EAItB,KAAK,iBAAiB,EACvB,MAAM,UAAU,CAAC;AA6BlB,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,UAAU,EAAE,IAAI,EAChB,YAAY,EAAE,MAAM,GAAG,IAAI,iBAuF5B;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,MAAM,EAAE,EACrB,UAAU,EAAE,IAAI,EAChB,YAAY,EAAE,MAAM,GAAG,IAAI,iBAK5B;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CASxE;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,iBAAiB,EAC1B,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,SAAS,GAAE,IAAiB,GAC3B,OAAO,CAAC,MAAM,CAAC,CA8HjB;AAED;;;GAGG;AACH,wBAAsB,mBAAmB,CACvC,KAAK,GAAE,MAA4B,EACnC,iBAAiB,CAAC,EAAE,MAAM,EAC1B,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAqC9B;AAED;;GAEG;AACH,wBAAgB,sCAAsC,IAAI,MAAM,EAAE,CAYjE;AAID;;GAEG;AACH,wBAAgB,8BAA8B,IAAI;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,CAqB7F;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,QAqB9C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,IAAI,UAExC;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,GACX;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAgBlE;AAED,wBAAsB,sCAAsC,kBAI3D;AAED,wBAAsB,UAAU,CAC9B,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAAC,CAY3E;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,MAAM,EACV,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,EAClC,QAAQ,CAAC,EAAE,aAAa,GACvB,OAAO,CAAC,IAAI,CAAC,CAuBf;AAED,wBAAsB,aAAa,CAAC,iBAAiB,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CAGhG;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAiD/B;AAED,wBAAgB,0BAA0B,CACxC,eAAe,EAAE,MAAM,EACvB,KAAK,GAAE,MAA4B,iCAOpC;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,QAAQ,EAAE,iCAI3D;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAuDlC;AAED,wBAAgB,UAAU,CAAC,KAAK,GAAE,MAA4B,iCAE7D;AAED,wBAAsB,YAAY,CAAC,KAAK,GAAE,MAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KAErE;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,2CAQnD;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;eAQpD;AAED,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CA0C7B;AAED,wBAAsB,QAAQ,CAAC,KAAK,GAAE,MAA4B,+BAEjE;AAED,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,yCAQ/C;AAED,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,iBAe9C;AAED;;;;GAIG;AACH,wBAAsB,cAAc,IAAI,OAAO,CAAC,IAAI,CAAC,CAQpD;AAED,wBAAsB,WAAW,CAC/B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAC/B,QAAQ,GAAE,MAAW,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAa5B;AAED,wBAAgB,WAAW,SAG1B;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,QAOnD;AAED,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,OAAO,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAElB,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACxC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACzC,CAAC;AAIF,wBAAsB,kBAAkB,CAAC,EACvC,KAA2B,EAC3B,GAAG,EACH,WAAW,GACZ,GAAE;IACD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,GAAG,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;CACjB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CA0FjC;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,SAAS,CASvF;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,EAC7D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,WAG9D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAMvF;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAsBvF;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,GACjB;IACD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAoCA;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,yBAAyB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,GAAG,IAAI,OA+BhG"}

package/dist/src/util/index.js

@@ -26,7 +26,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isJavascriptFile = isJavascriptFile;
 exports.writeOutput = writeOutput;
 exports.writeMultipleOutputs = writeMultipleOutputs;
 exports.readOutput = readOutput;
@@ -39,7 +38,6 @@ exports.filenameToDate = filenameToDate;
 exports.dateToFilename = dateToFilename;
 exports.readResult_fileSystem = readResult_fileSystem;
 exports.migrateResultsFromFileSystemToDatabase = migrateResultsFromFileSystemToDatabase;
-exports.cleanupOldFileResults = cleanupOldFileResults;
 exports.readResult = readResult;
 exports.updateResult = updateResult;
 exports.getLatestEval = getLatestEval;
@@ -88,21 +86,14 @@ const accounts_1 = require("../globalConfig/accounts");
 const googleSheets_1 = require("../googleSheets");
 const logger_1 = __importDefault(require("../logger"));
 const migrate_1 = require("../migrate");
+const eval_1 = __importStar(require("../models/eval"));
 const prompt_1 = require("../models/prompt");
 const types_1 = require("../types");
 const manage_1 = require("./config/manage");
 const createHash_1 = require("./createHash");
+const file_1 = require("./file");
 const templates_1 = require("./templates");
 const DEFAULT_QUERY_LIMIT = 100;
-/**
- * Checks if a file is a JavaScript or TypeScript file based on its extension.
- *
- * @param filePath - The path of the file to check.
- * @returns True if the file has a JavaScript or TypeScript extension, false otherwise.
- */
-function isJavascriptFile(filePath) {
-    return /\.(js|cjs|mjs|ts|cts|mts)$/.test(filePath);
-}
 const outputToSimpleString = (output) => {
     const passFailText = output.pass ? '[PASS]' : '[FAIL]';
     const namedScoresText = Object.entries(output.namedScores)
@@ -122,14 +113,16 @@ const outputToSimpleString = (output) => {
       ${gradingResultText}
     `.trim();
 };
-async function writeOutput(outputPath, evalId, results, config, shareableUrl) {
+async function writeOutput(outputPath, evalRecord, shareableUrl) {
+    const table = await evalRecord.getTable();
+    (0, tiny_invariant_1.default)(table, 'Table is required');
     if (outputPath.match(/^https:\/\/docs\.google\.com\/spreadsheets\//)) {
-        const rows = results.table.body.map((row) => {
+        const rows = table.body.map((row) => {
             const csvRow = {};
-            results.table.head.vars.forEach((varName, index) => {
+            table.head.vars.forEach((varName, index) => {
                 csvRow[varName] = row.vars[index];
             });
-            results.table.head.prompts.forEach((prompt, index) => {
+            table.head.prompts.forEach((prompt, index) => {
                 csvRow[prompt.label] = outputToSimpleString(row.outputs[index]);
             });
             return csvRow;
@@ -148,38 +141,51 @@ async function writeOutput(outputPath, evalId, results, config, shareableUrl) {
     if (outputExtension === 'csv') {
         const csvOutput = (0, sync_1.stringify)([
             [
-                ...results.table.head.vars,
-                ...results.table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`),
+                ...table.head.vars,
+                ...table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`),
             ],
-            ...results.table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)]),
+            ...table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)]),
         ]);
         fs.writeFileSync(outputPath, csvOutput);
     }
     else if (outputExtension === 'json') {
-        fs.writeFileSync(outputPath, JSON.stringify({ evalId, results, config, shareableUrl }, null, 2));
+        const summary = await evalRecord.toEvaluateSummary();
+        fs.writeFileSync(outputPath, JSON.stringify({
+            evalId: evalRecord.id,
+            results: summary,
+            config: evalRecord.config,
+            shareableUrl,
+        }, null, 2));
     }
     else if (outputExtension === 'yaml' || outputExtension === 'yml' || outputExtension === 'txt') {
-        fs.writeFileSync(outputPath, js_yaml_1.default.dump({ results, config, shareableUrl }));
+        const summary = await evalRecord.toEvaluateSummary();
+        fs.writeFileSync(outputPath, js_yaml_1.default.dump({
+            evalId: evalRecord.id,
+            results: summary,
+            config: evalRecord.config,
+            shareableUrl,
+        }));
     }
     else if (outputExtension === 'html') {
+        const summary = await evalRecord.toEvaluateSummary();
         const template = fs.readFileSync(`${(0, esm_1.getDirectory)()}/tableOutput.html`, 'utf-8');
-        const table = [
+        const htmlTable = [
             [
-                ...results.table.head.vars,
-                ...results.table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`),
+                ...table.head.vars,
+                ...table.head.prompts.map((prompt) => `[${prompt.provider}] ${prompt.label}`),
             ],
-            ...results.table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)]),
+            ...table.body.map((row) => [...row.vars, ...row.outputs.map(outputToSimpleString)]),
         ];
         const htmlOutput = (0, templates_1.getNunjucksEngine)().renderString(template, {
-            config,
-            table,
-            results: results.results,
+            config: evalRecord.config,
+            table: htmlTable,
+            results: summary,
         });
         fs.writeFileSync(outputPath, htmlOutput);
     }
 }
-async function writeMultipleOutputs(outputPaths, evalId, results, config, shareableUrl) {
-    await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalId, results, config, shareableUrl)));
+async function writeMultipleOutputs(outputPaths, evalRecord, shareableUrl) {
+    await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalRecord, shareableUrl)));
 }
 async function readOutput(outputPath) {
     const ext = path.parse(outputPath).ext.slice(1);
@@ -197,9 +203,9 @@ async function readOutput(outputPath) {
 function getLatestResultsPath() {
     return path.join((0, manage_1.getConfigDirectoryPath)(), 'output', 'latest.json');
 }
-async function writeResultsToDatabase(results, config, createdAt) {
+async function writeResultsToDatabase(results, config, createdAt = new Date()) {
     createdAt = createdAt || (results.timestamp ? new Date(results.timestamp) : new Date());
-    const evalId = `eval-${createdAt.toISOString().slice(0, 19)}`;
+    const evalId = (0, eval_1.createEvalId)(createdAt);
     const db = (0, database_1.getDb)();
     const promises = [];
     promises.push(db
@@ -216,9 +222,10 @@ async function writeResultsToDatabase(results, config, createdAt) {
         .run());
     logger_1.default.debug(`Inserting eval ${evalId}`);
     // Record prompt relation
+    (0, tiny_invariant_1.default)(results.table, 'Table is required');
     for (const prompt of results.table.head.prompts) {
         const label = prompt.label || prompt.display || prompt.raw;
-        const promptId = prompt.id || (0, prompt_1.generateIdFromPrompt)(prompt);
+        const promptId = (0, prompt_1.generateIdFromPrompt)(prompt);
         promises.push(db
             .insert(tables_1.prompts)
             .values({
@@ -297,7 +304,7 @@ async function writeResultsToDatabase(results, config, createdAt) {
  *
  * @returns Last n evals in descending order.
  */
-function listPreviousResults(limit = DEFAULT_QUERY_LIMIT, filterDescription, datasetId) {
+async function listPreviousResults(limit = DEFAULT_QUERY_LIMIT, filterDescription, datasetId) {
     const db = (0, database_1.getDb)();
     const startTime = performance.now();
     const query = db
@@ -310,7 +317,7 @@ function listPreviousResults(limit = DEFAULT_QUERY_LIMIT, filterDescription, dat
     })
         .from(tables_1.evals)
         .leftJoin(tables_1.evalsToDatasets, (0, drizzle_orm_1.eq)(tables_1.evals.id, tables_1.evalsToDatasets.evalId))
-        .where((0, drizzle_orm_1.and)(datasetId ? (0, drizzle_orm_1.eq)(tables_1.evalsToDatasets.datasetId, datasetId) : undefined, filterDescription ? (0, drizzle_orm_1.like)(tables_1.evals.description, `%${filterDescription}%`) : undefined));
+        .where((0, drizzle_orm_1.and)(datasetId ? (0, drizzle_orm_1.eq)(tables_1.evalsToDatasets.datasetId, datasetId) : undefined, filterDescription ? (0, drizzle_orm_1.like)(tables_1.evals.description, `%${filterDescription}%`) : undefined, (0, drizzle_orm_1.not)((0, drizzle_orm_1.eq)(tables_1.evals.results, {}))));
     const results = query.orderBy((0, drizzle_orm_1.desc)(tables_1.evals.createdAt)).limit(limit).all();
     const mappedResults = results.map((result) => ({
         evalId: result.evalId,
@@ -321,8 +328,10 @@ function listPreviousResults(limit = DEFAULT_QUERY_LIMIT, filterDescription, dat
     }));
     const endTime = performance.now();
     const executionTime = endTime - startTime;
+    const evalResults = await (0, eval_1.getSummaryofLatestEvals)(undefined, filterDescription, datasetId);
     logger_1.default.debug(`listPreviousResults execution time: ${executionTime.toFixed(2)}ms`);
-    return mappedResults;
+    const combinedResults = [...evalResults, ...mappedResults];
+    return combinedResults;
 }
 /**
  * @deprecated Used only for migration to sqlite
@@ -409,100 +418,19 @@ function readResult_fileSystem(name) {
         logger_1.default.error(`Failed to read results from ${resultsPath}:\n${err}`);
     }
 }
-let attemptedMigration = false;
 async function migrateResultsFromFileSystemToDatabase() {
-    if (attemptedMigration) {
-        // TODO(ian): Record this bit in the database.
-        return;
-    }
     // First run db migrations
     logger_1.default.debug('Running db migrations...');
     await (0, migrate_1.runDbMigrations)();
-    const fileNames = listPreviousResultFilenames_fileSystem();
-    if (fileNames.length === 0) {
-        return;
-    }
-    logger_1.default.info(`🔁 Migrating ${fileNames.length} flat files to local database.`);
-    logger_1.default.info('This is a one-time operation and may take a minute...');
-    attemptedMigration = true;
-    const outputDir = path.join((0, manage_1.getConfigDirectoryPath)(true /* createIfNotExists */), 'output');
-    const backupDir = `${outputDir}-backup-${new Date()
-        .toISOString()
-        .slice(0, 10)
-        .replace(/-/g, '')}`;
-    try {
-        fs.cpSync(outputDir, backupDir, { recursive: true });
-        logger_1.default.info(`Backup of output directory created at ${backupDir}`);
-    }
-    catch (backupError) {
-        logger_1.default.error(`Failed to create backup of output directory: ${backupError}`);
-        return;
-    }
-    logger_1.default.info('Moving files into database...');
-    const migrationPromises = fileNames.map(async (fileName) => {
-        const fileData = readResult_fileSystem(fileName);
-        if (fileData) {
-            await writeResultsToDatabase(fileData.result.results, fileData.result.config, filenameToDate(fileName));
-            logger_1.default.debug(`Migrated ${fileName} to database.`);
-            try {
-                fs.unlinkSync(path.join(outputDir, fileName));
-            }
-            catch (err) {
-                logger_1.default.warn(`Failed to delete ${fileName} after migration: ${err}`);
-            }
-        }
-        else {
-            logger_1.default.warn(`Failed to migrate result ${fileName} due to read error.`);
-        }
-    });
-    await Promise.all(migrationPromises);
-    try {
-        fs.unlinkSync(getLatestResultsPath());
-    }
-    catch (err) {
-        logger_1.default.warn(`Failed to delete latest.json: ${err}`);
-    }
-    logger_1.default.info('Migration complete. Please restart your web server if it is running.');
-}
-const RESULT_HISTORY_LENGTH = (0, envars_1.getEnvInt)('RESULT_HISTORY_LENGTH', DEFAULT_QUERY_LIMIT);
-function cleanupOldFileResults(remaining = RESULT_HISTORY_LENGTH) {
-    const sortedFilenames = listPreviousResultFilenames_fileSystem();
-    for (let i = 0; i < sortedFilenames.length - remaining; i++) {
-        fs.unlinkSync(path.join((0, manage_1.getConfigDirectoryPath)(), 'output', sortedFilenames[i]));
-    }
 }
 async function readResult(id) {
-    const db = (0, database_1.getDb)();
     try {
-        const evalResult = await db
-            .select({
-            id: tables_1.evals.id,
-            createdAt: tables_1.evals.createdAt,
-            author: tables_1.evals.author,
-            results: tables_1.evals.results,
-            config: tables_1.evals.config,
-            datasetId: tables_1.evalsToDatasets.datasetId,
-        })
-            .from(tables_1.evals)
-            .leftJoin(tables_1.evalsToDatasets, (0, drizzle_orm_1.eq)(tables_1.evals.id, tables_1.evalsToDatasets.evalId))
-            .where((0, drizzle_orm_1.eq)(tables_1.evals.id, id))
-            .execute();
-        if (evalResult.length === 0) {
-            return undefined;
-        }
-        const { id: resultId, createdAt, results, config, author, datasetId } = evalResult[0];
-        const result = {
-            version: 3,
-            createdAt: new Date(createdAt).toISOString().slice(0, 10),
-            author,
-            results,
-            config,
-            datasetId,
-        };
+        const eval_ = await eval_1.default.findById(id);
+        (0, tiny_invariant_1.default)(eval_, `Eval with ID ${id} not found.`);
         return {
-            id: resultId,
-            result,
-            createdAt: new Date(createdAt),
+            id,
+            result: await eval_.toResultsFile(),
+            createdAt: new Date(eval_.createdAt),
         };
     }
     catch (err) {
@@ -510,38 +438,20 @@ async function readResult(id) {
     }
 }
 async function updateResult(id, newConfig, newTable) {
-    const db = (0, database_1.getDb)();
     try {
         // Fetch the existing eval data from the database
-        const existingEval = await db
-            .select({
-            config: tables_1.evals.config,
-            results: tables_1.evals.results,
-        })
-            .from(tables_1.evals)
-            .where((0, drizzle_orm_1.eq)(tables_1.evals.id, id))
-            .limit(1)
-            .all();
-        if (existingEval.length === 0) {
+        const existingEval = await eval_1.default.findById(id);
+        if (!existingEval) {
             logger_1.default.error(`Eval with ID ${id} not found.`);
             return;
         }
-        const evalData = existingEval[0];
         if (newConfig) {
-            evalData.config = newConfig;
+            existingEval.config = newConfig;
         }
         if (newTable) {
-            evalData.results.table = newTable;
+            existingEval.setTable(newTable);
         }
-        await db
-            .update(tables_1.evals)
-            .set({
-            description: evalData.config.description,
-            config: evalData.config,
-            results: evalData.results,
-        })
-            .where((0, drizzle_orm_1.eq)(tables_1.evals.id, id))
-            .run();
+        await existingEval.save();
         logger_1.default.info(`Updated eval with ID ${id}`);
     }
     catch (err) {
@@ -549,61 +459,18 @@ async function updateResult(id, newConfig, newTable) {
     }
 }
 async function getLatestEval(filterDescription) {
-    const db = (0, database_1.getDb)();
-    let latestResults = await db
-        .select({
-        id: tables_1.evals.id,
-        createdAt: tables_1.evals.createdAt,
-        author: tables_1.evals.author,
-        description: tables_1.evals.description,
-        results: tables_1.evals.results,
-        config: tables_1.evals.config,
-    })
-        .from(tables_1.evals)
-        .orderBy((0, drizzle_orm_1.desc)(tables_1.evals.createdAt))
-        .limit(1);
-    if (filterDescription) {
-        const regex = new RegExp(filterDescription, 'i');
-        latestResults = latestResults.filter((result) => regex.test(result.description || ''));
-    }
-    if (!latestResults.length) {
-        return undefined;
-    }
-    const latestResult = latestResults[0];
-    return {
-        version: 3,
-        createdAt: new Date(latestResult.createdAt).toISOString(),
-        author: latestResult.author,
-        results: latestResult.results,
-        config: latestResult.config,
-    };
+    const eval_ = await eval_1.default.latest();
+    return await eval_?.toResultsFile();
 }
 async function getPromptsWithPredicate(predicate, limit) {
     // TODO(ian): Make this use a proper database query
-    const db = (0, database_1.getDb)();
-    const evals_ = await db
-        .select({
-        id: tables_1.evals.id,
-        createdAt: tables_1.evals.createdAt,
-        author: tables_1.evals.author,
-        results: tables_1.evals.results,
-        config: tables_1.evals.config,
-    })
-        .from(tables_1.evals)
-        .limit(limit)
-        .all();
+    const evals_ = await eval_1.default.getMany(limit);
     const groupedPrompts = {};
     for (const eval_ of evals_) {
         const createdAt = new Date(eval_.createdAt).toISOString();
-        const resultWrapper = {
-            version: 3,
-            createdAt,
-            author: eval_.author,
-            results: eval_.results,
-            config: eval_.config,
-        };
+        const resultWrapper = await eval_.toResultsFile();
         if (predicate(resultWrapper)) {
-            for (const prompt of resultWrapper.results.table.head.prompts) {
+            for (const prompt of eval_.getPrompts()) {
                 const promptId = (0, createHash_1.sha256)(prompt.raw);
                 const datasetId = resultWrapper.config.tests
                     ? (0, createHash_1.sha256)(JSON.stringify(resultWrapper.config.tests))
@@ -651,29 +518,11 @@ function getPromptsForTestCases(testCases) {
     return getPromptsForTestCasesHash(testCasesSha256);
 }
 async function getTestCasesWithPredicate(predicate, limit) {
-    const db = (0, database_1.getDb)();
-    const evals_ = await db
-        .select({
-        id: tables_1.evals.id,
-        createdAt: tables_1.evals.createdAt,
-        author: tables_1.evals.author,
-        results: tables_1.evals.results,
-        config: tables_1.evals.config,
-    })
-        .from(tables_1.evals)
-        .orderBy((0, drizzle_orm_1.desc)(tables_1.evals.createdAt))
-        .limit(limit)
-        .all();
+    const evals_ = await eval_1.default.getMany(limit);
     const groupedTestCases = {};
     for (const eval_ of evals_) {
         const createdAt = new Date(eval_.createdAt).toISOString();
-        const resultWrapper = {
-            version: 3,
-            createdAt,
-            author: eval_.author,
-            results: eval_.results,
-            config: eval_.config,
-        };
+        const resultWrapper = await eval_.toResultsFile();
         const testCases = resultWrapper.config.tests;
         if (testCases && predicate(resultWrapper)) {
             const evalId = eval_.id;
@@ -681,7 +530,7 @@ async function getTestCasesWithPredicate(predicate, limit) {
             if (datasetId in groupedTestCases) {
                 groupedTestCases[datasetId].recentEvalDate = new Date(Math.max(groupedTestCases[datasetId].recentEvalDate.getTime(), eval_.createdAt));
                 groupedTestCases[datasetId].count += 1;
-                const newPrompts = resultWrapper.results.table.head.prompts.map((prompt) => ({
+                const newPrompts = eval_.getPrompts().map((prompt) => ({
                     id: (0, createHash_1.sha256)(prompt.raw),
                     prompt,
                     evalId,
@@ -695,7 +544,7 @@ async function getTestCasesWithPredicate(predicate, limit) {
                 groupedTestCases[datasetId].prompts = Object.values(promptsById);
             }
             else {
-                const newPrompts = resultWrapper.results.table.head.prompts.map((prompt) => ({
+                const newPrompts = eval_.getPrompts().map((prompt) => ({
                     id: (0, createHash_1.sha256)(prompt.raw),
                     prompt,
                     evalId,
@@ -765,6 +614,7 @@ async function getEvalsWithPredicate(predicate, limit) {
             version: 3,
             createdAt,
             author: eval_.author,
+            // @ts-ignore
             results: eval_.results,
             config: eval_.config,
         };
@@ -774,6 +624,7 @@ async function getEvalsWithPredicate(predicate, limit) {
                 id: evalId,
                 date: new Date(eval_.createdAt),
                 config: eval_.config,
+                // @ts-ignore
                 results: eval_.results,
                 description: eval_.description || undefined,
             });
@@ -799,6 +650,9 @@ async function deleteEval(evalId) {
         // We need to clean up foreign keys first. We don't have onDelete: 'cascade' set on all these relationships.
         await db.delete(tables_1.evalsToPrompts).where((0, drizzle_orm_1.eq)(tables_1.evalsToPrompts.evalId, evalId)).run();
         await db.delete(tables_1.evalsToDatasets).where((0, drizzle_orm_1.eq)(tables_1.evalsToDatasets.evalId, evalId)).run();
+        await db.delete(tables_1.evalsToTags).where((0, drizzle_orm_1.eq)(tables_1.evalsToTags.evalId, evalId)).run();
+        await db.delete(tables_1.evalResultsTable).where((0, drizzle_orm_1.eq)(tables_1.evalResultsTable.evalId, evalId)).run();
+        await db.delete(tables_1.evalsToProviders).where((0, drizzle_orm_1.eq)(tables_1.evalsToProviders.evalId, evalId)).run();
         // Finally, delete the eval record
         const deletedIds = await db.delete(tables_1.evals).where((0, drizzle_orm_1.eq)(tables_1.evals.id, evalId)).run();
         if (deletedIds.changes === 0) {
@@ -848,7 +702,7 @@ function setupEnv(envPath) {
     }
 }
 const standaloneEvalCache = new node_cache_1.default({ stdTTL: 60 * 60 * 2 }); // Cache for 2 hours
-function getStandaloneEvals({ limit = DEFAULT_QUERY_LIMIT, tag, description, } = {}) {
+async function getStandaloneEvals({ limit = DEFAULT_QUERY_LIMIT, tag, description, } = {}) {
     const cacheKey = `standalone_evals_${limit}_${tag?.key}_${tag?.value}`;
     const cachedResult = standaloneEvalCache.get(cacheKey);
     if (cachedResult) {
@@ -876,16 +730,26 @@ function getStandaloneEvals({ limit = DEFAULT_QUERY_LIMIT, tag, description, } =
         .orderBy((0, drizzle_orm_1.desc)(tables_1.evals.createdAt))
         .limit(limit)
         .all();
-    const standaloneEvals = results.flatMap((result) => {
-        const { description, createdAt, evalId, promptId, datasetId, results: { table }, isRedteam, } = result;
-        return table.head.prompts.map((col, index) => {
+    const standaloneEvals = (await Promise.all(results.map(async (result) => {
+        const { description, createdAt, evalId, promptId, datasetId, 
+        // @ts-ignore
+        isRedteam, } = result;
+        const eval_ = await eval_1.default.findById(evalId);
+        (0, tiny_invariant_1.default)(eval_, `Eval with ID ${evalId} not found`);
+        const table = (await eval_.getTable()) || { body: [] };
+        // @ts-ignore
+        return eval_.getPrompts().map((col, index) => {
             // Compute some stats
-            const pluginCounts = table.body.reduce((acc, row) => {
+            const pluginCounts = table.body.reduce(
+            // @ts-ignore
+            (acc, row) => {
                 const pluginId = row.test.metadata?.pluginId;
                 if (pluginId) {
                     const isPass = row.outputs[index].pass;
-                    acc.pluginPassCount[pluginId] = (acc.pluginPassCount[pluginId] || 0) + (isPass ? 1 : 0);
-                    acc.pluginFailCount[pluginId] = (acc.pluginFailCount[pluginId] || 0) + (isPass ? 0 : 1);
+                    acc.pluginPassCount[pluginId] =
+                        (acc.pluginPassCount[pluginId] || 0) + (isPass ? 1 : 0);
+                    acc.pluginFailCount[pluginId] =
+                        (acc.pluginFailCount[pluginId] || 0) + (isPass ? 0 : 1);
                 }
                 return acc;
             }, { pluginPassCount: {}, pluginFailCount: {} });
@@ -900,7 +764,7 @@ function getStandaloneEvals({ limit = DEFAULT_QUERY_LIMIT, tag, description, } =
                 ...col,
             };
         });
-    });
+    }))).flat();
     standaloneEvalCache.set(cacheKey, standaloneEvals);
     return standaloneEvals;
 }
@@ -975,7 +839,7 @@ function parsePathOrGlob(basePath, promptPath) {
     let functionName;
     if (filename.includes(':')) {
         const splits = filename.split(':');
-        if (splits[0] && (isJavascriptFile(splits[0]) || splits[0].endsWith('.py'))) {
+        if (splits[0] && ((0, file_1.isJavascriptFile)(splits[0]) || splits[0].endsWith('.py'))) {
             [filename, functionName] = splits;
         }
     }