promptfoo 0.67.0 → 0.68.1

package/dist/src/util.d.ts

@@ -1,8 +1,8 @@
 import nunjucks from 'nunjucks';
 import { type EvalWithMetadata, type EvaluateResult, type EvaluateSummary, type EvaluateTable, type NunjucksFilterMap, type PromptWithMetadata, type ResultsFile, type TestCase, type TestCasesWithMetadata, type UnifiedConfig, type OutputFile, type Prompt, type CompletedPrompt } from './types';
-export declare function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined>;
 export declare function dereferenceConfig(rawConfig: UnifiedConfig): Promise<UnifiedConfig>;
 export declare function readConfig(configPath: string): Promise<UnifiedConfig>;
+export declare function maybeReadConfig(configPath: string): Promise<UnifiedConfig | undefined>;
 /**
  * Reads multiple configuration files and combines them into a single UnifiedConfig.
  *
@@ -10,8 +10,12 @@ export declare function readConfig(configPath: string): Promise<UnifiedConfig>;
  * @returns {Promise<UnifiedConfig>} A promise that resolves to a unified configuration object.
  */
 export declare function readConfigs(configPaths: string[]): Promise<UnifiedConfig>;
-export declare function writeMultipleOutputs(outputPaths: string[], evalId: string | null, results: EvaluateSummary, config: Partial<UnifiedConfig>, shareableUrl: string | null): Promise<void>;
+export declare function getNunjucksEngine(filters?: NunjucksFilterMap): nunjucks.Environment | {
+    renderString: (template: string) => string;
+};
 export declare function writeOutput(outputPath: string, evalId: string | null, results: EvaluateSummary, config: Partial<UnifiedConfig>, shareableUrl: string | null): Promise<void>;
+export declare function writeMultipleOutputs(outputPaths: string[], evalId: string | null, results: EvaluateSummary, config: Partial<UnifiedConfig>, shareableUrl: string | null): Promise<void>;
+export declare function sha256(str: string): string;
 export declare function readOutput(outputPath: string): Promise<OutputFile>;
 export declare function getConfigDirectoryPath(createIfNotExists?: boolean): string;
 export declare function setConfigDirectoryPath(newPath: string): void;
@@ -40,15 +44,8 @@ export declare function listPreviousResults_fileSystem(): {
     fileName: string;
     description?: string;
 }[];
-export declare function migrateResultsFromFileSystemToDatabase(): Promise<void>;
-export declare function cleanupOldFileResults(remaining?: number): void;
 export declare function filenameToDate(filename: string): Date;
 export declare function dateToFilename(date: Date): string;
-export declare function readResult(id: string): Promise<{
-    id: string;
-    result: ResultsFile;
-    createdAt: Date;
-} | undefined>;
 /**
  * @deprecated Used only for migration to sqlite
  */
@@ -57,25 +54,28 @@ export declare function readResult_fileSystem(name: string): {
     result: ResultsFile;
     createdAt: Date;
 } | undefined;
+export declare function migrateResultsFromFileSystemToDatabase(): Promise<void>;
+export declare function cleanupOldFileResults(remaining?: number): void;
+export declare function readResult(id: string): Promise<{
+    id: string;
+    result: ResultsFile;
+    createdAt: Date;
+} | undefined>;
 export declare function updateResult(id: string, newConfig?: Partial<UnifiedConfig>, newTable?: EvaluateTable): Promise<void>;
 export declare function readLatestResults(filterDescription?: string): Promise<ResultsFile | undefined>;
-export declare function getPromptsForTestCases(testCases: TestCase[]): Promise<PromptWithMetadata[]>;
+export declare function getPromptsWithPredicate(predicate: (result: ResultsFile) => boolean, limit: number): Promise<PromptWithMetadata[]>;
 export declare function getPromptsForTestCasesHash(testCasesSha256: string, limit?: number): Promise<PromptWithMetadata[]>;
-export declare function sha256(str: string): string;
+export declare function getPromptsForTestCases(testCases: TestCase[]): Promise<PromptWithMetadata[]>;
+export declare function getTestCasesWithPredicate(predicate: (result: ResultsFile) => boolean, limit: number): Promise<TestCasesWithMetadata[]>;
 export declare function getPrompts(limit?: number): Promise<PromptWithMetadata[]>;
-export declare function getPromptsWithPredicate(predicate: (result: ResultsFile) => boolean, limit: number): Promise<PromptWithMetadata[]>;
 export declare function getTestCases(limit?: number): Promise<TestCasesWithMetadata[]>;
-export declare function getTestCasesWithPredicate(predicate: (result: ResultsFile) => boolean, limit: number): Promise<TestCasesWithMetadata[]>;
 export declare function getPromptFromHash(hash: string): Promise<PromptWithMetadata | undefined>;
 export declare function getDatasetFromHash(hash: string): Promise<TestCasesWithMetadata | undefined>;
+export declare function getEvalsWithPredicate(predicate: (result: ResultsFile) => boolean, limit: number): Promise<EvalWithMetadata[]>;
 export declare function getEvals(limit?: number): Promise<EvalWithMetadata[]>;
 export declare function getEvalFromId(hash: string): Promise<EvalWithMetadata | undefined>;
-export declare function getEvalsWithPredicate(predicate: (result: ResultsFile) => boolean, limit: number): Promise<EvalWithMetadata[]>;
 export declare function deleteEval(evalId: string): Promise<void>;
 export declare function readFilters(filters: Record<string, string>): Promise<NunjucksFilterMap>;
-export declare function getNunjucksEngine(filters?: NunjucksFilterMap): nunjucks.Environment | {
-    renderString: (template: string) => string;
-};
 export declare function printBorder(): void;
 export declare function transformOutput(codeOrFilepath: string, output: string | object | undefined, context: {
     vars?: Record<string, string | object | undefined>;
@@ -94,4 +94,18 @@ export declare function resultIsForTestCase(result: EvaluateResult, testCase: Te
 export declare function safeJsonStringify(value: any, prettyPrint?: boolean): string;
 export declare function renderVarsInObject<T>(obj: T, vars?: Record<string, string | object>): T;
 export declare function extractJsonObjects(str: string): object[];
+/**
+ * Parses a file path or glob pattern to extract function names and file extensions.
+ * Function names can be specified in the filename like this:
+ * prompt.py:myFunction or prompts.js:myFunction.
+ * @param basePath - The base path for file resolution.
+ * @param promptPath - The path or glob pattern.
+ * @returns Parsed details including function name, file extension, and directory status.
+ */
+export declare function parsePathOrGlob(basePath: string, promptPath: string): {
+    extension?: string;
+    functionName?: string;
+    isPathPattern: boolean;
+    filePath: string;
+};
 //# sourceMappingURL=util.d.ts.map

package/dist/src/util.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/util.ts"],"names":[],"mappings":"AASA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAoBhC,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,aAAa,EAElB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,qBAAqB,EAE1B,KAAK,aAAa,EAClB,KAAK,UAAU,EAEf,KAAK,MAAM,EACX,KAAK,eAAe,EAIrB,MAAM,SAAS,CAAC;AAIjB,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAoGxF;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAe3E;AAED;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAkI/E;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,MAAM,EAAE,EACrB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAK5B;AAED,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAmF5B;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CASxE;AAID,wBAAgB,sBAAsB,CAAC,iBAAiB,GAAE,OAAe,GAAG,MAAM,CAMjF;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAE5D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,SAAS,CAAC,EAAE,IAAI,GACf,OAAO,CAAC,MAAM,CAAC,CA2FjB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,GAAE,MAA4B,EACnC,iBAAiB,CAAC,EAAE,MAAM,GACzB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,EAAE,CAqBnD;AAED;;GAEG;AACH,wBAAgB,sCAAsC,IAAI,MAAM,EAAE,CAYjE;AAID;;GAEG;AACH,wBAAgB,8BAA8B,IAAI;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,CAqB7F;AAGD,wBAAsB,sCAAsC,kBA0D3D;AAKD,wBAAgB,qBAAqB,CAAC,SAAS,SAAwB,QAKtE;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,QAqB9C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,IAAI,UAExC;AAED,wBAAsB,UAAU,CAC9B,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAAC,CAiC3E;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,GACX;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAgBlE;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,MAAM,EACV,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,EAClC,QAAQ,CAAC,EAAE,aAAa,GACvB,OAAO,CAAC,IAAI,CAAC,CAyCf;AAED,wBAAsB,iBAAiB,CACrC,iBAAiB,CAAC,EAAE,MAAM,GACzB,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CA8BlC;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,QAAQ,EAAE,iCAI3D;AAED,wBAAgB,0BAA0B,CACxC,eAAe,EAAE,MAAM,EACvB,KAAK,GAAE,MAA4B,iCAOpC;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,MAAM,UAEjC;AAED,wBAAgB,UAAU,CAAC,KAAK,GAAE,MAA4B,iCAE7D;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAgE/B;AAED,wBAAsB,YAAY,CAAC,KAAK,GAAE,MAA4B,oCAErE;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAqElC;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,2CAQnD;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,8CAQpD;AAED,wBAAsB,QAAQ,CAAC,KAAK,GAAE,MAA4B,+BAEjE;AAED,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,yCAQ/C;AAED,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAsC7B;AAED,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,iBAa9C;AAED,wBAAsB,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAc7F;AAED,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,iBAAiB;6BAG9B,MAAM;EAcpC;AAED,wBAAgB,WAAW,SAG1B;AAED,wBAAsB,eAAe,CACnC,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,EACnC,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;CAAE,gBA0CzF;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,QAOnD;AAED,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,KAAK,GAAE,MAA4B,GAAG,cAAc,EAAE,CA+BxF;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,SAAS,CAQvF;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,EAC7D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,WAG9D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAMvF;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,GAAG,EAAE,WAAW,GAAE,OAAe,GAAG,MAAM,CAelF;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAsBvF;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CA4BxD"}
+{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../src/util.ts"],"names":[],"mappings":"AASA,OAAO,QAAQ,MAAM,UAAU,CAAC;AAqBhC,OAAO,EACL,KAAK,gBAAgB,EACrB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,aAAa,EAElB,KAAK,iBAAiB,EACtB,KAAK,kBAAkB,EACvB,KAAK,WAAW,EAChB,KAAK,QAAQ,EACb,KAAK,qBAAqB,EAE1B,KAAK,aAAa,EAClB,KAAK,UAAU,EAEf,KAAK,MAAM,EACX,KAAK,eAAe,EAIrB,MAAM,SAAS,CAAC;AAIjB,wBAAsB,iBAAiB,CAAC,SAAS,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC,CAoGxF;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,CAe3E;AAED,wBAAsB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC,CAK5F;AAED;;;;;GAKG;AACH,wBAAsB,WAAW,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAkI/E;AAED,wBAAgB,iBAAiB,CAAC,OAAO,CAAC,EAAE,iBAAiB;6BAG9B,MAAM;EAcpC;AAED,wBAAsB,WAAW,CAC/B,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAmF5B;AAED,wBAAsB,oBAAoB,CACxC,WAAW,EAAE,MAAM,EAAE,EACrB,MAAM,EAAE,MAAM,GAAG,IAAI,EACrB,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,YAAY,EAAE,MAAM,GAAG,IAAI,iBAK5B;AAED,wBAAgB,MAAM,CAAC,GAAG,EAAE,MAAM,UAEjC;AAED,wBAAsB,UAAU,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC,CASxE;AAID,wBAAgB,sBAAsB,CAAC,iBAAiB,GAAE,OAAe,GAAG,MAAM,CAMjF;AAED,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAE5D;AAED;;;GAGG;AACH,wBAAgB,oBAAoB,IAAI,MAAM,CAE7C;AAED,wBAAsB,sBAAsB,CAC1C,OAAO,EAAE,eAAe,EACxB,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,EAC9B,SAAS,CAAC,EAAE,IAAI,GACf,OAAO,CAAC,MAAM,CAAC,CA2FjB;AAED;;;GAGG;AACH,wBAAgB,mBAAmB,CACjC,KAAK,GAAE,MAA4B,EACnC,iBAAiB,CAAC,EAAE,MAAM,GACzB;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;CAAE,EAAE,CAqBnD;AAED;;GAEG;AACH,wBAAgB,sCAAsC,IAAI,MAAM,EAAE,CAYjE;AAID;;GAEG;AACH,wBAAgB,8BAA8B,IAAI;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,WAAW,CAAC,EAAE,MAAM,CAAA;CAAE,EAAE,CAqB7F;AAED,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,QAqB9C;AAED,wBAAgB,cAAc,CAAC,IAAI,EAAE,IAAI,UAExC;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,MAAM,GACX;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAgBlE;AAID,wBAAsB,sCAAsC,kBA0D3D;AAKD,wBAAgB,qBAAqB,CAAC,SAAS,SAAwB,QAKtE;AAED,wBAAsB,UAAU,CAC9B,EAAE,EAAE,MAAM,GACT,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,WAAW,CAAC;IAAC,SAAS,EAAE,IAAI,CAAA;CAAE,GAAG,SAAS,CAAC,CAiC3E;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,MAAM,EACV,SAAS,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,EAClC,QAAQ,CAAC,EAAE,aAAa,GACvB,OAAO,CAAC,IAAI,CAAC,CAyCf;AAED,wBAAsB,iBAAiB,CACrC,iBAAiB,CAAC,EAAE,MAAM,GACzB,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC,CA8BlC;AAED,wBAAsB,uBAAuB,CAC3C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAgE/B;AAED,wBAAgB,0BAA0B,CACxC,eAAe,EAAE,MAAM,EACvB,KAAK,GAAE,MAA4B,iCAOpC;AAED,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,QAAQ,EAAE,iCAI3D;AAED,wBAAsB,yBAAyB,CAC7C,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,qBAAqB,EAAE,CAAC,CAqElC;AAED,wBAAgB,UAAU,CAAC,KAAK,GAAE,MAA4B,iCAE7D;AAED,wBAAsB,YAAY,CAAC,KAAK,GAAE,MAA4B,oCAErE;AAED,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,2CAQnD;AAED,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,8CAQpD;AAED,wBAAsB,qBAAqB,CACzC,SAAS,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,EAC3C,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAsC7B;AAED,wBAAsB,QAAQ,CAAC,KAAK,GAAE,MAA4B,+BAEjE;AAED,wBAAsB,aAAa,CAAC,IAAI,EAAE,MAAM,yCAQ/C;AAED,wBAAsB,UAAU,CAAC,MAAM,EAAE,MAAM,iBAa9C;AAED,wBAAsB,WAAW,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAc7F;AAED,wBAAgB,WAAW,SAG1B;AAED,wBAAsB,eAAe,CACnC,cAAc,EAAE,MAAM,EACtB,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,EACnC,OAAO,EAAE;IAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS,CAAC,CAAC;IAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;CAAE,gBA0CzF;AAED,wBAAgB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,QAOnD;AAED,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;CACzB,CAAC;AAEF,wBAAgB,kBAAkB,CAAC,KAAK,GAAE,MAA4B,GAAG,cAAc,EAAE,CA+BxF;AAED,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,CAAC,UAAU,CAAC,GAAG,MAAM,GAAG,SAAS,CAQvF;AAED,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,EAC7D,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,EAAE,GAAG,MAAM,CAAC,GAAG,SAAS,WAG9D;AAED,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,cAAc,EAAE,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAMvF;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,GAAG,EAAE,WAAW,GAAE,OAAe,GAAG,MAAM,CAelF;AAED,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAsBvF;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,EAAE,CA4BxD;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,GACjB;IACD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,OAAO,CAAC;IACvB,QAAQ,EAAE,MAAM,CAAC;CAClB,CAoCA"}

package/dist/src/util.js

@@ -26,7 +26,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.extractJsonObjects = exports.renderVarsInObject = exports.safeJsonStringify = exports.resultIsForTestCase = exports.varsMatch = exports.providerToIdentifier = exports.getStandaloneEvals = exports.setupEnv = exports.transformOutput = exports.printBorder = exports.getNunjucksEngine = exports.readFilters = exports.deleteEval = exports.getEvalsWithPredicate = exports.getEvalFromId = exports.getEvals = exports.getDatasetFromHash = exports.getPromptFromHash = exports.getTestCasesWithPredicate = exports.getTestCases = exports.getPromptsWithPredicate = exports.getPrompts = exports.sha256 = exports.getPromptsForTestCasesHash = exports.getPromptsForTestCases = exports.readLatestResults = exports.updateResult = exports.readResult_fileSystem = exports.readResult = exports.dateToFilename = exports.filenameToDate = exports.cleanupOldFileResults = exports.migrateResultsFromFileSystemToDatabase = exports.listPreviousResults_fileSystem = exports.listPreviousResultFilenames_fileSystem = exports.listPreviousResults = exports.writeResultsToDatabase = exports.getLatestResultsPath = exports.setConfigDirectoryPath = exports.getConfigDirectoryPath = exports.readOutput = exports.writeOutput = exports.writeMultipleOutputs = exports.readConfigs = exports.readConfig = exports.dereferenceConfig = exports.maybeReadConfig = void 0;
+exports.parsePathOrGlob = exports.extractJsonObjects = exports.renderVarsInObject = exports.safeJsonStringify = exports.resultIsForTestCase = exports.varsMatch = exports.providerToIdentifier = exports.getStandaloneEvals = exports.setupEnv = exports.transformOutput = exports.printBorder = exports.readFilters = exports.deleteEval = exports.getEvalFromId = exports.getEvals = exports.getEvalsWithPredicate = exports.getDatasetFromHash = exports.getPromptFromHash = exports.getTestCases = exports.getPrompts = exports.getTestCasesWithPredicate = exports.getPromptsForTestCases = exports.getPromptsForTestCasesHash = exports.getPromptsWithPredicate = exports.readLatestResults = exports.updateResult = exports.readResult = exports.cleanupOldFileResults = exports.migrateResultsFromFileSystemToDatabase = exports.readResult_fileSystem = exports.dateToFilename = exports.filenameToDate = exports.listPreviousResults_fileSystem = exports.listPreviousResultFilenames_fileSystem = exports.listPreviousResults = exports.writeResultsToDatabase = exports.getLatestResultsPath = exports.setConfigDirectoryPath = exports.getConfigDirectoryPath = exports.readOutput = exports.sha256 = exports.writeMultipleOutputs = exports.writeOutput = exports.getNunjucksEngine = exports.readConfigs = exports.maybeReadConfig = exports.readConfig = exports.dereferenceConfig = void 0;
 const json_schema_ref_parser_1 = __importDefault(require("@apidevtools/json-schema-ref-parser"));
 const crypto_1 = require("crypto");
 const sync_1 = require("csv-stringify/sync");
@@ -41,6 +41,7 @@ const os = __importStar(require("os"));
 const path = __importStar(require("path"));
 const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
 const cliState_1 = __importDefault(require("./cliState"));
+const constants_1 = require("./constants");
 const database_1 = require("./database");
 const esm_1 = require("./esm");
 const googleSheets_1 = require("./googleSheets");
@@ -50,13 +51,6 @@ const wrapper_1 = require("./python/wrapper");
 const testCases_1 = require("./testCases");
 const types_1 = require("./types");
 const DEFAULT_QUERY_LIMIT = 100;
-async function maybeReadConfig(configPath) {
-    if (!fs.existsSync(configPath)) {
-        return undefined;
-    }
-    return readConfig(configPath);
-}
-exports.maybeReadConfig = maybeReadConfig;
 async function dereferenceConfig(rawConfig) {
     if (process.env.PROMPTFOO_DISABLE_REF_PARSER) {
         return rawConfig;
@@ -159,6 +153,13 @@ async function readConfig(configPath) {
     }
 }
 exports.readConfig = readConfig;
+async function maybeReadConfig(configPath) {
+    if (!fs.existsSync(configPath)) {
+        return undefined;
+    }
+    return readConfig(configPath);
+}
+exports.maybeReadConfig = maybeReadConfig;
 /**
  * Reads multiple configuration files and combines them into a single UnifiedConfig.
  *
@@ -290,10 +291,23 @@ async function readConfigs(configPaths) {
     return combinedConfig;
 }
 exports.readConfigs = readConfigs;
-async function writeMultipleOutputs(outputPaths, evalId, results, config, shareableUrl) {
-    await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalId, results, config, shareableUrl)));
+function getNunjucksEngine(filters) {
+    if (process.env.PROMPTFOO_DISABLE_TEMPLATING) {
+        return {
+            renderString: (template) => template,
+        };
+    }
+    const env = nunjucks_1.default.configure({
+        autoescape: false,
+    });
+    if (filters) {
+        for (const [name, filter] of Object.entries(filters)) {
+            env.addFilter(name, filter);
+        }
+    }
+    return env;
 }
-exports.writeMultipleOutputs = writeMultipleOutputs;
+exports.getNunjucksEngine = getNunjucksEngine;
 async function writeOutput(outputPath, evalId, results, config, shareableUrl) {
     const outputExtension = outputPath.split('.').pop()?.toLowerCase();
     const outputToSimpleString = (output) => {
@@ -372,6 +386,14 @@ ${gradingResultText}`.trim();
     }
 }
 exports.writeOutput = writeOutput;
+async function writeMultipleOutputs(outputPaths, evalId, results, config, shareableUrl) {
+    await Promise.all(outputPaths.map((outputPath) => writeOutput(outputPath, evalId, results, config, shareableUrl)));
+}
+exports.writeMultipleOutputs = writeMultipleOutputs;
+function sha256(str) {
+    return (0, crypto_1.createHash)('sha256').update(str).digest('hex');
+}
+exports.sha256 = sha256;
 async function readOutput(outputPath) {
     const ext = path.parse(outputPath).ext.slice(1);
     switch (ext) {
@@ -545,6 +567,51 @@ function listPreviousResults_fileSystem() {
     });
 }
 exports.listPreviousResults_fileSystem = listPreviousResults_fileSystem;
+function filenameToDate(filename) {
+    const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
+    // Replace hyphens with colons where necessary (Windows compatibility).
+    const dateParts = dateString.split('T');
+    const timePart = dateParts[1].replace(/-/g, ':');
+    const formattedDateString = `${dateParts[0]}T${timePart}`;
+    const date = new Date(formattedDateString);
+    return date;
+    /*
+    return date.toLocaleDateString('en-US', {
+      year: 'numeric',
+      month: 'long',
+      day: 'numeric',
+      hour: '2-digit',
+      minute: '2-digit',
+      second: '2-digit',
+      timeZoneName: 'short',
+    });
+    */
+}
+exports.filenameToDate = filenameToDate;
+function dateToFilename(date) {
+    return `eval-${date.toISOString().replace(/:/g, '-')}.json`;
+}
+exports.dateToFilename = dateToFilename;
+/**
+ * @deprecated Used only for migration to sqlite
+ */
+function readResult_fileSystem(name) {
+    const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
+    const resultsPath = path.join(resultsDirectory, name);
+    try {
+        const result = JSON.parse(fs.readFileSync(fs.realpathSync(resultsPath), 'utf-8'));
+        const createdAt = filenameToDate(name);
+        return {
+            id: sha256(JSON.stringify(result.config)),
+            result,
+            createdAt,
+        };
+    }
+    catch (err) {
+        logger_1.default.error(`Failed to read results from ${resultsPath}:\n${err}`);
+    }
+}
+exports.readResult_fileSystem = readResult_fileSystem;
 let attemptedMigration = false;
 async function migrateResultsFromFileSystemToDatabase() {
     if (attemptedMigration) {
@@ -609,31 +676,6 @@ function cleanupOldFileResults(remaining = RESULT_HISTORY_LENGTH) {
     }
 }
 exports.cleanupOldFileResults = cleanupOldFileResults;
-function filenameToDate(filename) {
-    const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
-    // Replace hyphens with colons where necessary (Windows compatibility).
-    const dateParts = dateString.split('T');
-    const timePart = dateParts[1].replace(/-/g, ':');
-    const formattedDateString = `${dateParts[0]}T${timePart}`;
-    const date = new Date(formattedDateString);
-    return date;
-    /*
-    return date.toLocaleDateString('en-US', {
-      year: 'numeric',
-      month: 'long',
-      day: 'numeric',
-      hour: '2-digit',
-      minute: '2-digit',
-      second: '2-digit',
-      timeZoneName: 'short',
-    });
-    */
-}
-exports.filenameToDate = filenameToDate;
-function dateToFilename(date) {
-    return `eval-${date.toISOString().replace(/:/g, '-')}.json`;
-}
-exports.dateToFilename = dateToFilename;
 async function readResult(id) {
     const db = (0, database_1.getDb)();
     try {
@@ -668,26 +710,6 @@ async function readResult(id) {
     }
 }
 exports.readResult = readResult;
-/**
- * @deprecated Used only for migration to sqlite
- */
-function readResult_fileSystem(name) {
-    const resultsDirectory = path.join(getConfigDirectoryPath(), 'output');
-    const resultsPath = path.join(resultsDirectory, name);
-    try {
-        const result = JSON.parse(fs.readFileSync(fs.realpathSync(resultsPath), 'utf-8'));
-        const createdAt = filenameToDate(name);
-        return {
-            id: sha256(JSON.stringify(result.config)),
-            result,
-            createdAt,
-        };
-    }
-    catch (err) {
-        logger_1.default.error(`Failed to read results from ${resultsPath}:\n${err}`);
-    }
-}
-exports.readResult_fileSystem = readResult_fileSystem;
 async function updateResult(id, newConfig, newTable) {
     const db = (0, database_1.getDb)();
     try {
@@ -757,28 +779,6 @@ async function readLatestResults(filterDescription) {
     };
 }
 exports.readLatestResults = readLatestResults;
-function getPromptsForTestCases(testCases) {
-    const testCasesJson = JSON.stringify(testCases);
-    const testCasesSha256 = sha256(testCasesJson);
-    return getPromptsForTestCasesHash(testCasesSha256);
-}
-exports.getPromptsForTestCases = getPromptsForTestCases;
-function getPromptsForTestCasesHash(testCasesSha256, limit = DEFAULT_QUERY_LIMIT) {
-    return getPromptsWithPredicate((result) => {
-        const testsJson = JSON.stringify(result.config.tests);
-        const hash = sha256(testsJson);
-        return hash === testCasesSha256;
-    }, limit);
-}
-exports.getPromptsForTestCasesHash = getPromptsForTestCasesHash;
-function sha256(str) {
-    return (0, crypto_1.createHash)('sha256').update(str).digest('hex');
-}
-exports.sha256 = sha256;
-function getPrompts(limit = DEFAULT_QUERY_LIMIT) {
-    return getPromptsWithPredicate(() => true, limit);
-}
-exports.getPrompts = getPrompts;
 async function getPromptsWithPredicate(predicate, limit) {
     // TODO(ian): Make this use a proper database query
     const db = (0, database_1.getDb)();
@@ -838,10 +838,20 @@ async function getPromptsWithPredicate(predicate, limit) {
     return Object.values(groupedPrompts);
 }
 exports.getPromptsWithPredicate = getPromptsWithPredicate;
-async function getTestCases(limit = DEFAULT_QUERY_LIMIT) {
-    return getTestCasesWithPredicate(() => true, limit);
+function getPromptsForTestCasesHash(testCasesSha256, limit = DEFAULT_QUERY_LIMIT) {
+    return getPromptsWithPredicate((result) => {
+        const testsJson = JSON.stringify(result.config.tests);
+        const hash = sha256(testsJson);
+        return hash === testCasesSha256;
+    }, limit);
 }
-exports.getTestCases = getTestCases;
+exports.getPromptsForTestCasesHash = getPromptsForTestCasesHash;
+function getPromptsForTestCases(testCases) {
+    const testCasesJson = JSON.stringify(testCases);
+    const testCasesSha256 = sha256(testCasesJson);
+    return getPromptsForTestCasesHash(testCasesSha256);
+}
+exports.getPromptsForTestCases = getPromptsForTestCases;
 async function getTestCasesWithPredicate(predicate, limit) {
     const db = (0, database_1.getDb)();
     const evals_ = await db
@@ -909,6 +919,14 @@ async function getTestCasesWithPredicate(predicate, limit) {
     return Object.values(groupedTestCases);
 }
 exports.getTestCasesWithPredicate = getTestCasesWithPredicate;
+function getPrompts(limit = DEFAULT_QUERY_LIMIT) {
+    return getPromptsWithPredicate(() => true, limit);
+}
+exports.getPrompts = getPrompts;
+async function getTestCases(limit = DEFAULT_QUERY_LIMIT) {
+    return getTestCasesWithPredicate(() => true, limit);
+}
+exports.getTestCases = getTestCases;
 async function getPromptFromHash(hash) {
     const prompts = await getPrompts();
     for (const prompt of prompts) {
@@ -929,20 +947,6 @@ async function getDatasetFromHash(hash) {
     return undefined;
 }
 exports.getDatasetFromHash = getDatasetFromHash;
-async function getEvals(limit = DEFAULT_QUERY_LIMIT) {
-    return getEvalsWithPredicate(() => true, limit);
-}
-exports.getEvals = getEvals;
-async function getEvalFromId(hash) {
-    const evals_ = await getEvals();
-    for (const eval_ of evals_) {
-        if (eval_.id.startsWith(hash)) {
-            return eval_;
-        }
-    }
-    return undefined;
-}
-exports.getEvalFromId = getEvalFromId;
 async function getEvalsWithPredicate(predicate, limit) {
     const db = (0, database_1.getDb)();
     const evals_ = await db
@@ -980,6 +984,20 @@ async function getEvalsWithPredicate(predicate, limit) {
     return ret;
 }
 exports.getEvalsWithPredicate = getEvalsWithPredicate;
+async function getEvals(limit = DEFAULT_QUERY_LIMIT) {
+    return getEvalsWithPredicate(() => true, limit);
+}
+exports.getEvals = getEvals;
+async function getEvalFromId(hash) {
+    const evals_ = await getEvals();
+    for (const eval_ of evals_) {
+        if (eval_.id.startsWith(hash)) {
+            return eval_;
+        }
+    }
+    return undefined;
+}
+exports.getEvalFromId = getEvalFromId;
 async function deleteEval(evalId) {
     const db = (0, database_1.getDb)();
     await db.transaction(async () => {
@@ -1010,25 +1028,8 @@ async function readFilters(filters) {
     return ret;
 }
 exports.readFilters = readFilters;
-function getNunjucksEngine(filters) {
-    if (process.env.PROMPTFOO_DISABLE_TEMPLATING) {
-        return {
-            renderString: (template) => template,
-        };
-    }
-    const env = nunjucks_1.default.configure({
-        autoescape: false,
-    });
-    if (filters) {
-        for (const [name, filter] of Object.entries(filters)) {
-            env.addFilter(name, filter);
-        }
-    }
-    return env;
-}
-exports.getNunjucksEngine = getNunjucksEngine;
 function printBorder() {
-    const border = '='.repeat((process.stdout.columns || 80) - 10);
+    const border = '='.repeat(constants_1.TERMINAL_MAX_WIDTH);
     logger_1.default.info(border);
 }
 exports.printBorder = printBorder;
@@ -1201,4 +1202,44 @@ function extractJsonObjects(str) {
     return jsonObjects;
 }
 exports.extractJsonObjects = extractJsonObjects;
+/**
+ * Parses a file path or glob pattern to extract function names and file extensions.
+ * Function names can be specified in the filename like this:
+ * prompt.py:myFunction or prompts.js:myFunction.
+ * @param basePath - The base path for file resolution.
+ * @param promptPath - The path or glob pattern.
+ * @returns Parsed details including function name, file extension, and directory status.
+ */
+function parsePathOrGlob(basePath, promptPath) {
+    let filePath = path.join(basePath, promptPath);
+    if (filePath.includes('file:')) {
+        filePath = filePath.split('file:')[1];
+    }
+    let stats;
+    try {
+        stats = fs.statSync(filePath);
+    }
+    catch (err) {
+        if (process.env.PROMPTFOO_STRICT_FILES) {
+            throw err;
+        }
+    }
+    let filename = path.relative(basePath, filePath);
+    let functionName;
+    if (filename.includes(':')) {
+        const splits = filename.split(':');
+        if (splits[0] && ['.js', '.cjs', '.mjs', '.py'].some((ext) => splits[0].endsWith(ext))) {
+            [filename, functionName] = splits;
+        }
+    }
+    const isPathPattern = stats?.isDirectory() || /[*?{}\[\]]/.test(filePath); // glob pattern
+    const safeFilename = path.relative(basePath, path.isAbsolute(filename) ? filename : path.resolve(basePath, filename));
+    return {
+        extension: isPathPattern ? undefined : path.parse(safeFilename).ext,
+        filePath: safeFilename.startsWith(basePath) ? safeFilename : path.join(basePath, safeFilename),
+        functionName,
+        isPathPattern,
+    };
+}
+exports.parsePathOrGlob = parsePathOrGlob;
 //# sourceMappingURL=util.js.map