promptfoo 0.65.2 → 0.67.0

package/dist/src/prompts/index.js

@@ -0,0 +1,143 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.readPrompts = exports.processPrompt = exports.readProviderPromptMap = void 0;
+const glob_1 = require("glob");
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+const logger_1 = __importDefault(require("../logger"));
+const javascript_1 = require("./processors/javascript");
+const json_1 = require("./processors/json");
+const jsonl_1 = require("./processors/jsonl");
+const python_1 = require("./processors/python");
+const string_1 = require("./processors/string");
+const text_1 = require("./processors/text");
+const yaml_1 = require("./processors/yaml");
+const utils_1 = require("./utils");
+__exportStar(require("./grading"), exports);
+/**
+ * Reads and maps provider prompts based on the configuration and parsed prompts.
+ * @param config - The configuration object.
+ * @param parsedPrompts - Array of parsed prompts.
+ * @returns A map of provider IDs to their respective prompts.
+ */
+function readProviderPromptMap(config, parsedPrompts) {
+    const ret = {};
+    if (!config.providers) {
+        return ret;
+    }
+    const allPrompts = [];
+    for (const prompt of parsedPrompts) {
+        allPrompts.push(prompt.label);
+    }
+    if (typeof config.providers === 'string') {
+        return { [config.providers]: allPrompts };
+    }
+    if (typeof config.providers === 'function') {
+        return { 'Custom function': allPrompts };
+    }
+    for (const provider of config.providers) {
+        if (typeof provider === 'object') {
+            // It's either a ProviderOptionsMap or a ProviderOptions
+            if (provider.id) {
+                const rawProvider = provider;
+                (0, tiny_invariant_1.default)(rawProvider.id, 'You must specify an `id` on the Provider when you override options.prompts');
+                ret[rawProvider.id] = rawProvider.prompts || allPrompts;
+                if (rawProvider.label) {
+                    ret[rawProvider.label] = rawProvider.prompts || allPrompts;
+                }
+            }
+            else {
+                const rawProvider = provider;
+                const originalId = Object.keys(rawProvider)[0];
+                const providerObject = rawProvider[originalId];
+                const id = providerObject.id || originalId;
+                ret[id] = rawProvider[originalId].prompts || allPrompts;
+            }
+        }
+    }
+    return ret;
+}
+exports.readProviderPromptMap = readProviderPromptMap;
+/**
+ * Processes a raw prompt based on its content type and path.
+ * @param prompt - The raw prompt data.
+ * @param basePath - Base path for file resolution.
+ * @param maxRecursionDepth - Maximum recursion depth for globbing.
+ * @returns Promise resolving to an array of processed prompts.
+ */
+async function processPrompt(prompt, basePath = '', maxRecursionDepth = 1) {
+    (0, tiny_invariant_1.default)(typeof prompt.raw === 'string', `prompt.raw must be a string, but got ${JSON.stringify(prompt.raw)}`);
+    if (!(0, utils_1.maybeFilePath)(prompt.raw)) {
+        return (0, string_1.processString)(prompt);
+    }
+    const { extension, functionName, isPathPattern, promptPath, } = (0, utils_1.parsePathOrGlob)(basePath, prompt.raw);
+    if (isPathPattern && maxRecursionDepth > 0) {
+        const globbedPath = (0, glob_1.globSync)(promptPath.replace(/\\/g, '/'), {
+            windowsPathsNoEscape: true,
+        });
+        logger_1.default.debug(`Expanded prompt ${prompt.raw} to ${promptPath} and then to ${JSON.stringify(globbedPath)}`);
+        const prompts = [];
+        for (const globbedFilePath of globbedPath) {
+            const processedPrompts = await processPrompt({ raw: globbedFilePath }, basePath, maxRecursionDepth - 1);
+            prompts.push(...processedPrompts);
+        }
+        return prompts;
+    }
+    if (extension === '.json') {
+        return (0, json_1.processJsonFile)(promptPath, prompt);
+    }
+    if (extension === '.jsonl') {
+        return (0, jsonl_1.processJsonlFile)(promptPath, prompt);
+    }
+    if (['.js', '.cjs', '.mjs'].includes(extension)) {
+        return (0, javascript_1.processJsFile)(promptPath, prompt, functionName);
+    }
+    if (extension === '.py') {
+        return (0, python_1.processPythonFile)(promptPath, prompt, functionName);
+    }
+    if (extension === '.txt') {
+        return (0, text_1.processTxtFile)(promptPath, prompt);
+    }
+    if (['.yml', '.yaml'].includes(extension)) {
+        return (0, yaml_1.processYamlFile)(promptPath, prompt);
+    }
+    return [];
+}
+exports.processPrompt = processPrompt;
+/**
+ * Reads and processes prompts from a specified path or glob pattern.
+ * @param promptPathOrGlobs - The path or glob pattern.
+ * @param basePath - Base path for file resolution.
+ * @returns Promise resolving to an array of processed prompts.
+ */
+async function readPrompts(promptPathOrGlobs, basePath = '') {
+    logger_1.default.debug(`Reading prompts from ${JSON.stringify(promptPathOrGlobs)}`);
+    const promptPartials = (0, utils_1.normalizeInput)(promptPathOrGlobs);
+    const prompts = [];
+    for (const prompt of promptPartials) {
+        const promptBatch = await processPrompt(prompt, basePath);
+        if (promptBatch.length === 0) {
+            throw new Error(`There are no prompts in ${JSON.stringify(prompt.raw)}`);
+        }
+        prompts.push(...promptBatch);
+    }
+    return prompts;
+}
+exports.readPrompts = readPrompts;
+//# sourceMappingURL=index.js.map

package/dist/src/prompts/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/prompts/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,+BAAgC;AAChC,oEAAuC;AACvC,uDAA+B;AAQ/B,wDAAwD;AACxD,4CAAoD;AACpD,8CAAsD;AACtD,gDAAwD;AACxD,gDAAoD;AACpD,4CAAmD;AACnD,4CAAoD;AACpD,mCAAyE;AAEzE,4CAA0B;AAE1B;;;;;GAKG;AACH,SAAgB,qBAAqB,CACnC,MAA8B,EAC9B,aAAuB;IAEvB,MAAM,GAAG,GAA6B,EAAE,CAAC;IAEzC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QACtB,OAAO,GAAG,CAAC;IACb,CAAC;IAED,MAAM,UAAU,GAAG,EAAE,CAAC;IACtB,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;QACnC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,QAAQ,EAAE,CAAC;QACzC,OAAO,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,UAAU,EAAE,CAAC;IAC5C,CAAC;IAED,IAAI,OAAO,MAAM,CAAC,SAAS,KAAK,UAAU,EAAE,CAAC;QAC3C,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,CAAC;IAC3C,CAAC;IAED,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;QACxC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE,CAAC;YACjC,wDAAwD;YACxD,IAAI,QAAQ,CAAC,EAAE,EAAE,CAAC;gBAChB,MAAM,WAAW,GAAG,QAA2B,CAAC;gBAChD,IAAA,wBAAS,EACP,WAAW,CAAC,EAAE,EACd,4EAA4E,CAC7E,CAAC;gBACF,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,OAAO,IAAI,UAAU,CAAC;gBACxD,IAAI,WAAW,CAAC,KAAK,EAAE,CAAC;oBACtB,GAAG,CAAC,WAAW,CAAC,KAAK,CAAC,GAAG,WAAW,CAAC,OAAO,IAAI,UAAU,CAAC;gBAC7D,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,WAAW,GAAG,QAA8B,CAAC;gBACnD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC/C,MAAM,cAAc,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC;gBAC/C,MAAM,EAAE,GAAG,cAAc,CAAC,EAAE,IAAI,UAAU,CAAC;gBAC3C,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,UAAU,CAAC,CAAC,OAAO,IAAI,UAAU,CAAC;YAC1D,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AA/CD,sDA+CC;AAED;;;;;;GAMG;AACI,KAAK,UAAU,aAAa,CACjC,MAAuB,EACvB,WAAmB,EAAE,EACrB,oBAA4B,CAAC;IAE7B,IAAA,wBAAS,EACP,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,EAC9B,wCAAwC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACrE,CAAC;IAEF,IAAI,CAAC,IAAA,qBAAa,EAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAA,sBAAa,EAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,EACJ,SAAS,EACT,YAAY,EACZ,aAAa,EACb,UAAU,GACX,GAKG,IAAA,uBAAe,EAAC,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC;IAE1C,IAAI,aAAa,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC3C,MAAM,WAAW,GAAG,IAAA,eAAQ,EAAC,UAAU,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE;YAC3D,oBAAoB,EAAE,IAAI;SAC3B,CAAC,CAAC;QACH,gBAAM,CAAC,KAAK,CACV,mBAAmB,MAAM,CAAC,GAAG,OAAO,UAAU,gBAAgB,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAC5F,CAAC;QACF,MAAM,OAAO,GAAa,EAAE,CAAC;QAC7B,KAAK,MAAM,eAAe,IAAI,WAAW,EAAE,CAAC;YAC1C,MAAM,gBAAgB,GAAG,MAAM,aAAa,CAC1C,EAAE,GAAG,EAAE,eAAe,EAAE,EACxB,QAAQ,EACR,iBAAiB,GAAG,CAAC,CACtB,CAAC;YACF,OAAO,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;QACpC,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,IAAI,SAAS,KAAK,OAAO,EAAE,CAAC;QAC1B,OAAO,IAAA,sBAAe,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,SAAS,KAAK,QAAQ,EAAE,CAAC;QAC3B,OAAO,IAAA,wBAAgB,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAChD,OAAO,IAAA,0BAAa,EAAC,UAAU,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IACzD,CAAC;IACD,IAAI,SAAS,KAAK,KAAK,EAAE,CAAC;QACxB,OAAO,IAAA,0BAAiB,EAAC,UAAU,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;QACzB,OAAO,IAAA,qBAAc,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;QAC1C,OAAO,IAAA,sBAAe,EAAC,UAAU,EAAE,MAAM,CAAC,CAAC;IAC7C,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAhED,sCAgEC;AAED;;;;;GAKG;AACI,KAAK,UAAU,WAAW,CAC/B,iBAAiF,EACjF,WAAmB,EAAE;IAErB,gBAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;IAE1E,MAAM,cAAc,GAAsB,IAAA,sBAAc,EAAC,iBAAiB,CAAC,CAAC;IAC5E,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,MAAM,IAAI,cAAc,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,MAAM,aAAa,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QAC1D,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,GAAG,WAAW,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAhBD,kCAgBC"}

package/dist/src/prompts/processors/javascript.d.ts

@@ -0,0 +1,9 @@
+import { Prompt } from '../../types';
+/**
+ * Processes a JavaScript file to import and execute a module function as a prompt.
+ * @param filePath - Path to the JavaScript file.
+ * @param functionName - Optional function name to execute.
+ * @returns Promise resolving to an array of prompts.
+ */
+export declare function processJsFile(filePath: string, prompt: Partial<Prompt>, functionName: string | undefined): Promise<Prompt[]>;
+//# sourceMappingURL=javascript.d.ts.map

package/dist/src/prompts/processors/javascript.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"javascript.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/javascript.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;;;;GAKG;AACH,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EACvB,YAAY,EAAE,MAAM,GAAG,SAAS,GAC/B,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB"}

package/dist/src/prompts/processors/javascript.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processJsFile = void 0;
+const esm_1 = require("../../esm");
+/**
+ * Processes a JavaScript file to import and execute a module function as a prompt.
+ * @param filePath - Path to the JavaScript file.
+ * @param functionName - Optional function name to execute.
+ * @returns Promise resolving to an array of prompts.
+ */
+async function processJsFile(filePath, prompt, functionName) {
+    const promptFunction = await (0, esm_1.importModule)(filePath, functionName);
+    return [
+        {
+            raw: String(promptFunction),
+            label: prompt.label ? prompt.label : functionName ? `${filePath}:${functionName}` : filePath,
+            function: promptFunction,
+        },
+    ];
+}
+exports.processJsFile = processJsFile;
+//# sourceMappingURL=javascript.js.map

package/dist/src/prompts/processors/javascript.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"javascript.js","sourceRoot":"","sources":["../../../../src/prompts/processors/javascript.ts"],"names":[],"mappings":";;;AAAA,mCAAyC;AAGzC;;;;;GAKG;AACI,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,MAAuB,EACvB,YAAgC;IAEhC,MAAM,cAAc,GAAG,MAAM,IAAA,kBAAY,EAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAClE,OAAO;QACL;YACE,GAAG,EAAE,MAAM,CAAC,cAAc,CAAC;YAC3B,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,GAAG,QAAQ,IAAI,YAAY,EAAE,CAAC,CAAC,CAAC,QAAQ;YAC5F,QAAQ,EAAE,cAAc;SACzB;KACF,CAAC;AACJ,CAAC;AAbD,sCAaC"}

package/dist/src/prompts/processors/json.d.ts

@@ -0,0 +1,12 @@
+import { Prompt } from '../../types';
+/**
+ * Processes a JSON file to extract prompts.
+ * This function reads a JSON file and converts it to a `Prompt` object.
+ *
+ * @param filePath - The path to the JSON file.
+ * @param prompt - The raw prompt data, used for labeling.
+ * @returns An array of one `Prompt` object.
+ * @throws Will throw an error if the file cannot be read.
+ */
+export declare function processJsonFile(filePath: string, prompt: Partial<Prompt>): Prompt[];
+//# sourceMappingURL=json.d.ts.map

package/dist/src/prompts/processors/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/json.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;;;;;;;GAQG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CASnF"}

package/dist/src/prompts/processors/json.js

@@ -0,0 +1,48 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processJsonFile = void 0;
+const fs = __importStar(require("fs"));
+/**
+ * Processes a JSON file to extract prompts.
+ * This function reads a JSON file and converts it to a `Prompt` object.
+ *
+ * @param filePath - The path to the JSON file.
+ * @param prompt - The raw prompt data, used for labeling.
+ * @returns An array of one `Prompt` object.
+ * @throws Will throw an error if the file cannot be read.
+ */
+function processJsonFile(filePath, prompt) {
+    const fileContents = fs.readFileSync(filePath, 'utf8');
+    // NOTE: We do not validate if this is a valid JSON file.
+    return [
+        {
+            raw: fileContents,
+            label: prompt.label || `${filePath}: ${fileContents}`,
+        },
+    ];
+}
+exports.processJsonFile = processJsonFile;
+//# sourceMappingURL=json.js.map

package/dist/src/prompts/processors/json.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.js","sourceRoot":"","sources":["../../../../src/prompts/processors/json.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AAGzB;;;;;;;;GAQG;AACH,SAAgB,eAAe,CAAC,QAAgB,EAAE,MAAuB;IACvE,MAAM,YAAY,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACvD,yDAAyD;IACzD,OAAO;QACL;YACE,GAAG,EAAE,YAAY;YACjB,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG,QAAQ,KAAK,YAAY,EAAE;SACtD;KACF,CAAC;AACJ,CAAC;AATD,0CASC"}

package/dist/src/prompts/processors/jsonl.d.ts

@@ -0,0 +1,9 @@
+import { Prompt } from '../../types';
+/**
+ * Processes a JSONL file to extract prompts.
+ * @param filePath - Path to the JSONL file.
+ * @param prompt - The raw prompt data.
+ * @returns Array of prompts extracted from the file.
+ */
+export declare function processJsonlFile(filePath: string, prompt: Partial<Prompt>): Prompt[];
+//# sourceMappingURL=jsonl.d.ts.map

package/dist/src/prompts/processors/jsonl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/jsonl.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAYpF"}

package/dist/src/prompts/processors/jsonl.js

@@ -0,0 +1,48 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processJsonlFile = void 0;
+const fs = __importStar(require("fs"));
+/**
+ * Processes a JSONL file to extract prompts.
+ * @param filePath - Path to the JSONL file.
+ * @param prompt - The raw prompt data.
+ * @returns Array of prompts extracted from the file.
+ */
+function processJsonlFile(filePath, prompt) {
+    const fileContent = fs.readFileSync(filePath, 'utf-8');
+    const jsonLines = fileContent.split(/\r?\n/).filter((line) => line.length > 0);
+    const containsMultiple = jsonLines.length > 1;
+    return jsonLines.map((json) => ({
+        raw: json,
+        label: containsMultiple
+            ? prompt.label
+                ? `${prompt.label}: ${json}`
+                : `${filePath}: ${json}`
+            : prompt.label || `${filePath}`,
+    }));
+}
+exports.processJsonlFile = processJsonlFile;
+//# sourceMappingURL=jsonl.js.map

package/dist/src/prompts/processors/jsonl.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonl.js","sourceRoot":"","sources":["../../../../src/prompts/processors/jsonl.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AAGzB;;;;;GAKG;AACH,SAAgB,gBAAgB,CAAC,QAAgB,EAAE,MAAuB;IACxE,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC/E,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9C,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QAC9B,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,gBAAgB;YACrB,CAAC,CAAC,MAAM,CAAC,KAAK;gBACZ,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE;gBAC5B,CAAC,CAAC,GAAG,QAAQ,KAAK,IAAI,EAAE;YAC1B,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,GAAG,QAAQ,EAAE;KAClC,CAAC,CAAC,CAAC;AACN,CAAC;AAZD,4CAYC"}

package/dist/src/prompts/processors/python.d.ts

@@ -0,0 +1,31 @@
+import type { Prompt, ApiProvider } from '../../types';
+/**
+ * Python prompt function. Runs a specific function from the python file.
+ * @param promptPath - Path to the Python file.
+ * @param functionName - Function name to execute.
+ * @param context - Context for the prompt.
+ * @returns The prompts
+ */
+export declare const pythonPromptFunction: (filePath: string, functionName: string, context: {
+    vars: Record<string, string | object>;
+    provider?: ApiProvider;
+}) => Promise<any>;
+/**
+ * Legacy Python prompt function. Runs the whole python file.
+ * @param filePath - Path to the Python file.
+ * @param context - Context for the prompt.
+ * @returns The prompts
+ */
+export declare const pythonPromptFunctionLegacy: (filePath: string, context: {
+    vars: Record<string, string | object>;
+    provider?: ApiProvider;
+}) => Promise<string>;
+/**
+ * Processes a Python file to extract or execute a function as a prompt.
+ * @param filePath - Path to the Python file.
+ * @param prompt - The raw prompt data.
+ * @param functionName - Optional function name to execute.
+ * @returns Array of prompts extracted or executed from the file.
+ */
+export declare function processPythonFile(filePath: string, prompt: Partial<Prompt>, functionName: string | undefined): Prompt[];
+//# sourceMappingURL=python.d.ts.map

package/dist/src/prompts/processors/python.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"python.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/python.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAGvD;;;;;;GAMG;AACH,eAAO,MAAM,oBAAoB,aACrB,MAAM,gBACF,MAAM,WACX;IACP,IAAI,EAAE,OAAO,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB,iBAWF,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,0BAA0B,aAC3B,MAAM,WACP;IACP,IAAI,EAAE,OAAO,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;IACtC,QAAQ,CAAC,EAAE,WAAW,CAAC;CACxB,oBAWF,CAAC;AAEF;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,EACvB,YAAY,EAAE,MAAM,GAAG,SAAS,GAC/B,MAAM,EAAE,CAeV"}

package/dist/src/prompts/processors/python.js

@@ -0,0 +1,96 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processPythonFile = exports.pythonPromptFunctionLegacy = exports.pythonPromptFunction = void 0;
+const fs = __importStar(require("fs"));
+const python_shell_1 = require("python-shell");
+const logger_1 = __importDefault(require("../../logger"));
+const wrapper_1 = require("../../python/wrapper");
+const util_1 = require("../../util");
+/**
+ * Python prompt function. Runs a specific function from the python file.
+ * @param promptPath - Path to the Python file.
+ * @param functionName - Function name to execute.
+ * @param context - Context for the prompt.
+ * @returns The prompts
+ */
+const pythonPromptFunction = async (filePath, functionName, context) => {
+    return (0, wrapper_1.runPython)(filePath, functionName, [
+        {
+            ...context,
+            provider: {
+                id: context.provider?.id,
+                label: context.provider?.label,
+            },
+        },
+    ]);
+};
+exports.pythonPromptFunction = pythonPromptFunction;
+/**
+ * Legacy Python prompt function. Runs the whole python file.
+ * @param filePath - Path to the Python file.
+ * @param context - Context for the prompt.
+ * @returns The prompts
+ */
+const pythonPromptFunctionLegacy = async (filePath, context) => {
+    const options = {
+        mode: 'text',
+        pythonPath: process.env.PROMPTFOO_PYTHON || 'python',
+        args: [(0, util_1.safeJsonStringify)(context)],
+    };
+    logger_1.default.debug(`Executing python prompt script ${filePath}`);
+    const results = (await python_shell_1.PythonShell.run(filePath, options)).join('\n');
+    logger_1.default.debug(`Python prompt script ${filePath} returned: ${results}`);
+    return results;
+};
+exports.pythonPromptFunctionLegacy = pythonPromptFunctionLegacy;
+/**
+ * Processes a Python file to extract or execute a function as a prompt.
+ * @param filePath - Path to the Python file.
+ * @param prompt - The raw prompt data.
+ * @param functionName - Optional function name to execute.
+ * @returns Array of prompts extracted or executed from the file.
+ */
+function processPythonFile(filePath, prompt, functionName) {
+    const fileContent = fs.readFileSync(filePath, 'utf-8');
+    return [
+        {
+            raw: fileContent,
+            label: prompt.label
+                ? prompt.label
+                : functionName
+                    ? `${filePath}:${functionName}`
+                    : `${filePath}: ${fileContent}`,
+            function: functionName
+                ? (...args) => (0, exports.pythonPromptFunction)(filePath, functionName, ...args)
+                : (...args) => (0, exports.pythonPromptFunctionLegacy)(filePath, ...args),
+        },
+    ];
+}
+exports.processPythonFile = processPythonFile;
+//# sourceMappingURL=python.js.map

package/dist/src/prompts/processors/python.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"python.js","sourceRoot":"","sources":["../../../../src/prompts/processors/python.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AACzB,+CAA0E;AAC1E,0DAAkC;AAClC,kDAAiD;AAEjD,qCAA+C;AAE/C;;;;;;GAMG;AACI,MAAM,oBAAoB,GAAG,KAAK,EACvC,QAAgB,EAChB,YAAoB,EACpB,OAGC,EACD,EAAE;IACF,OAAO,IAAA,mBAAS,EAAC,QAAQ,EAAE,YAAY,EAAE;QACvC;YACE,GAAG,OAAO;YACV,QAAQ,EAAE;gBACR,EAAE,EAAE,OAAO,CAAC,QAAQ,EAAE,EAAE;gBACxB,KAAK,EAAE,OAAO,CAAC,QAAQ,EAAE,KAAK;aAC/B;SACF;KACF,CAAC,CAAC;AACL,CAAC,CAAC;AAjBW,QAAA,oBAAoB,wBAiB/B;AAEF;;;;;GAKG;AACI,MAAM,0BAA0B,GAAG,KAAK,EAC7C,QAAgB,EAChB,OAGC,EACD,EAAE;IACF,MAAM,OAAO,GAAuB;QAClC,IAAI,EAAE,MAAM;QACZ,UAAU,EAAE,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,QAAQ;QACpD,IAAI,EAAE,CAAC,IAAA,wBAAiB,EAAC,OAAO,CAAC,CAAC;KACnC,CAAC;IACF,gBAAM,CAAC,KAAK,CAAC,kCAAkC,QAAQ,EAAE,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,CAAC,MAAM,0BAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtE,gBAAM,CAAC,KAAK,CAAC,wBAAwB,QAAQ,cAAc,OAAO,EAAE,CAAC,CAAC;IACtE,OAAO,OAAO,CAAC;AACjB,CAAC,CAAC;AAhBW,QAAA,0BAA0B,8BAgBrC;AAEF;;;;;;GAMG;AACH,SAAgB,iBAAiB,CAC/B,QAAgB,EAChB,MAAuB,EACvB,YAAgC;IAEhC,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvD,OAAO;QACL;YACE,GAAG,EAAE,WAAW;YAChB,KAAK,EAAE,MAAM,CAAC,KAAK;gBACjB,CAAC,CAAC,MAAM,CAAC,KAAK;gBACd,CAAC,CAAC,YAAY;oBACZ,CAAC,CAAC,GAAG,QAAQ,IAAI,YAAY,EAAE;oBAC/B,CAAC,CAAC,GAAG,QAAQ,KAAK,WAAW,EAAE;YACnC,QAAQ,EAAE,YAAY;gBACpB,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAA,4BAAoB,EAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,IAAI,CAAC;gBACpE,CAAC,CAAC,CAAC,GAAG,IAAI,EAAE,EAAE,CAAC,IAAA,kCAA0B,EAAC,QAAQ,EAAE,GAAG,IAAI,CAAC;SAC/D;KACF,CAAC;AACJ,CAAC;AAnBD,8CAmBC"}

package/dist/src/prompts/processors/string.d.ts

@@ -0,0 +1,8 @@
+import { Prompt } from '../../types';
+/**
+ * Processes a string as a literal prompt.
+ * @param prompt - The raw prompt data.
+ * @returns Array of prompts created from the string.
+ */
+export declare function processString(prompt: Partial<Prompt>): Prompt[];
+//# sourceMappingURL=string.d.ts.map

package/dist/src/prompts/processors/string.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"string.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/string.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAW/D"}

package/dist/src/prompts/processors/string.js

@@ -0,0 +1,23 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processString = void 0;
+const tiny_invariant_1 = __importDefault(require("tiny-invariant"));
+/**
+ * Processes a string as a literal prompt.
+ * @param prompt - The raw prompt data.
+ * @returns Array of prompts created from the string.
+ */
+function processString(prompt) {
+    (0, tiny_invariant_1.default)(typeof prompt.raw === 'string', `prompt.raw must be a string, but got ${JSON.stringify(prompt.raw)}`);
+    return [
+        {
+            raw: prompt.raw,
+            label: prompt.label ?? `${prompt.raw}`,
+        },
+    ];
+}
+exports.processString = processString;
+//# sourceMappingURL=string.js.map

package/dist/src/prompts/processors/string.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"string.js","sourceRoot":"","sources":["../../../../src/prompts/processors/string.ts"],"names":[],"mappings":";;;;;;AAAA,oEAAuC;AAGvC;;;;GAIG;AACH,SAAgB,aAAa,CAAC,MAAuB;IACnD,IAAA,wBAAS,EACP,OAAO,MAAM,CAAC,GAAG,KAAK,QAAQ,EAC9B,wCAAwC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACrE,CAAC;IACF,OAAO;QACL;YACE,GAAG,EAAE,MAAM,CAAC,GAAG;YACf,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE;SACvC;KACF,CAAC;AACJ,CAAC;AAXD,sCAWC"}

package/dist/src/prompts/processors/text.d.ts

@@ -0,0 +1,9 @@
+import { Prompt } from '../../types';
+/**
+ * Processes a text file to extract prompts, splitting by a delimiter.
+ * @param filePath - Path to the text file.
+ * @param prompt - The raw prompt data.
+ * @returns Array of prompts extracted from the file.
+ */
+export declare function processTxtFile(filePath: string, { label }: Partial<Prompt>): Prompt[];
+//# sourceMappingURL=text.d.ts.map

package/dist/src/prompts/processors/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/text.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAGrC;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAUrF"}

package/dist/src/prompts/processors/text.js

@@ -0,0 +1,47 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processTxtFile = void 0;
+const fs = __importStar(require("fs"));
+const constants_1 = require("../constants");
+/**
+ * Processes a text file to extract prompts, splitting by a delimiter.
+ * @param filePath - Path to the text file.
+ * @param prompt - The raw prompt data.
+ * @returns Array of prompts extracted from the file.
+ */
+function processTxtFile(filePath, { label }) {
+    const fileContent = fs.readFileSync(filePath, 'utf-8');
+    return fileContent // handle leading/trailing delimiters and empty lines
+        .split(constants_1.PROMPT_DELIMITER)
+        .map((p) => p.trim())
+        .filter((p) => p.length > 0)
+        .map((prompt) => ({
+        raw: prompt,
+        label: label ? `${label}: ${filePath}: ${prompt}` : `${filePath}: ${prompt}`,
+    }));
+}
+exports.processTxtFile = processTxtFile;
+//# sourceMappingURL=text.js.map

package/dist/src/prompts/processors/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../../../../src/prompts/processors/text.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,uCAAyB;AAEzB,4CAAgD;AAEhD;;;;;GAKG;AACH,SAAgB,cAAc,CAAC,QAAgB,EAAE,EAAE,KAAK,EAAmB;IACzE,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACvD,OAAO,WAAW,CAAC,qDAAqD;SACrE,KAAK,CAAC,4BAAgB,CAAC;SACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QAChB,GAAG,EAAE,MAAM;QACX,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,QAAQ,KAAK,MAAM,EAAE,CAAC,CAAC,CAAC,GAAG,QAAQ,KAAK,MAAM,EAAE;KAC7E,CAAC,CAAC,CAAC;AACR,CAAC;AAVD,wCAUC"}

package/dist/src/prompts/processors/yaml.d.ts

@@ -0,0 +1,13 @@
+import { Prompt } from '../../types';
+/**
+ * Processes a YAML file to extract prompts.
+ * This function reads a YAML file, parses it, and maps each entry to a `Prompt` object.
+ * Each prompt is labeled with the file path and the YAML content.
+ *
+ * @param filePath - The path to the YAML file.
+ * @param prompt - The raw prompt data, used for labeling.
+ * @returns An array of `Prompt` objects extracted from the YAML file.
+ * @throws Will throw an error if the file cannot be read or parsed.
+ */
+export declare function processYamlFile(filePath: string, prompt: Partial<Prompt>): Prompt[];
+//# sourceMappingURL=yaml.d.ts.map

package/dist/src/prompts/processors/yaml.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"yaml.d.ts","sourceRoot":"","sources":["../../../../src/prompts/processors/yaml.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAErC;;;;;;;;;GASG;AACH,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,GAAG,MAAM,EAAE,CAWnF"}