promptfoo 0.18.0 → 0.18.1

package/README.md

@@ -4,15 +4,15 @@
 [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/typpo/promptfoo/main.yml)](https://github.com/typpo/promptfoo/actions/workflows/main.yml)
 ![MIT license](https://img.shields.io/github/license/typpo/promptfoo)
 
-`promptfoo` is a tool for testing and evaluating LLM prompt quality.
+`promptfoo` is a tool for testing and evaluating LLM output quality.
 
 With promptfoo, you can:
 
-- **Systematically test prompts** against predefined test cases
+- **Systematically test prompts & models** against predefined test cases
 - **Evaluate quality and catch regressions** by comparing LLM outputs side-by-side
-- **Speed up evaluations** with caching and concurrent tests
-- **Score outputs automatically** by defining "expectations"
-- Use as a CLI, or integrate into your workflow as a library
+- **Speed up evaluations** with caching and concurrency
+- **Score outputs automatically** by defining test cases
+- Use as a CLI, library, or in CI/CD
 - Use OpenAI, Anthropic, open-source models like Llama and Vicuna, or integrate custom API providers for any LLM API
 
 The goal: **test-driven prompt engineering**, rather than trial-and-error.

package/dist/package.json

@@ -2,7 +2,7 @@
     "name": "promptfoo",
     "description": "LLM eval & testing toolkit",
     "author": "Ian Webster",
-    "version": "0.18.0",
+    "version": "0.18.1",
     "license": "MIT",
     "type": "commonjs",
     "main": "dist/src/index.js",

package/dist/src/providers/llama.d.ts

@@ -0,0 +1,30 @@
+import type { ApiProvider, ProviderResponse } from '../types.js';
+interface LlamaCompletionOptions {
+    n_predict?: number;
+    temperature?: number;
+    top_k?: number;
+    top_p?: number;
+    n_keep?: number;
+    stop?: string[];
+    repeat_penalty?: number;
+    repeat_last_n?: number;
+    penalize_nl?: boolean;
+    presence_penalty?: number;
+    frequency_penalty?: number;
+    mirostat?: boolean;
+    mirostat_tau?: number;
+    mirostat_eta?: number;
+    seed?: number;
+    ignore_eos?: boolean;
+    logit_bias?: Record<string, number>;
+}
+export declare class LlamaProvider implements ApiProvider {
+    modelName: string;
+    options?: LlamaCompletionOptions;
+    constructor(modelName: string, options?: LlamaCompletionOptions);
+    id(): string;
+    toString(): string;
+    callApi(prompt: string, options?: LlamaCompletionOptions): Promise<ProviderResponse>;
+}
+export {};
+//# sourceMappingURL=llama.d.ts.map

package/dist/src/providers/llama.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama.d.ts","sourceRoot":"","sources":["../../../src/providers/llama.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAEjE,UAAU,sBAAsB;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC;IAChB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACrC;AAED,qBAAa,aAAc,YAAW,WAAW;IAC/C,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,sBAAsB,CAAC;gBAErB,SAAS,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB;IAK/D,EAAE,IAAI,MAAM;IAIZ,QAAQ,IAAI,MAAM;IAIZ,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,gBAAgB,CAAC;CAoD3F"}

package/dist/src/providers/llama.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LlamaProvider = void 0;
+const cache_1 = require("../cache");
+const shared_1 = require("./shared");
+class LlamaProvider {
+    constructor(modelName, options) {
+        this.modelName = modelName;
+        this.options = options;
+    }
+    id() {
+        return `llama:${this.modelName}`;
+    }
+    toString() {
+        return `[Llama Provider ${this.modelName}]`;
+    }
+    async callApi(prompt, options) {
+        options = Object.assign({}, this.options, options);
+        const body = {
+            prompt,
+            n_predict: options?.n_predict || 512,
+            temperature: options?.temperature,
+            top_k: options?.top_k,
+            top_p: options?.top_p,
+            n_keep: options?.n_keep,
+            stop: options?.stop,
+            repeat_penalty: options?.repeat_penalty,
+            repeat_last_n: options?.repeat_last_n,
+            penalize_nl: options?.penalize_nl,
+            presence_penalty: options?.presence_penalty,
+            frequency_penalty: options?.frequency_penalty,
+            mirostat: options?.mirostat,
+            mirostat_tau: options?.mirostat_tau,
+            mirostat_eta: options?.mirostat_eta,
+            seed: options?.seed,
+            ignore_eos: options?.ignore_eos,
+            logit_bias: options?.logit_bias,
+        };
+        let response;
+        try {
+            response = await (0, cache_1.fetchJsonWithCache)(`${process.env.LLAMA_BASE_URL || 'http://localhost:8080'}/completion`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                },
+                body: JSON.stringify(body),
+            }, shared_1.REQUEST_TIMEOUT_MS);
+        }
+        catch (err) {
+            return {
+                error: `API call error: ${String(err)}`,
+            };
+        }
+        try {
+            return {
+                output: response.data.content,
+            };
+        }
+        catch (err) {
+            return {
+                error: `API response error: ${String(err)}: ${JSON.stringify(response.data)}`,
+            };
+        }
+    }
+}
+exports.LlamaProvider = LlamaProvider;
+//# sourceMappingURL=llama.js.map

package/dist/src/providers/llama.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llama.js","sourceRoot":"","sources":["../../../src/providers/llama.ts"],"names":[],"mappings":";;;AAAA,oCAA8C;AAC9C,qCAA8C;AAwB9C,MAAa,aAAa;IAIxB,YAAY,SAAiB,EAAE,OAAgC;QAC7D,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;IAED,EAAE;QACA,OAAO,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC;IAED,QAAQ;QACN,OAAO,mBAAmB,IAAI,CAAC,SAAS,GAAG,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,MAAc,EAAE,OAAgC;QAC5D,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QACnD,MAAM,IAAI,GAAG;YACX,MAAM;YACN,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,GAAG;YACpC,WAAW,EAAE,OAAO,EAAE,WAAW;YACjC,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,KAAK,EAAE,OAAO,EAAE,KAAK;YACrB,MAAM,EAAE,OAAO,EAAE,MAAM;YACvB,IAAI,EAAE,OAAO,EAAE,IAAI;YACnB,cAAc,EAAE,OAAO,EAAE,cAAc;YACvC,aAAa,EAAE,OAAO,EAAE,aAAa;YACrC,WAAW,EAAE,OAAO,EAAE,WAAW;YACjC,gBAAgB,EAAE,OAAO,EAAE,gBAAgB;YAC3C,iBAAiB,EAAE,OAAO,EAAE,iBAAiB;YAC7C,QAAQ,EAAE,OAAO,EAAE,QAAQ;YAC3B,YAAY,EAAE,OAAO,EAAE,YAAY;YACnC,YAAY,EAAE,OAAO,EAAE,YAAY;YACnC,IAAI,EAAE,OAAO,EAAE,IAAI;YACnB,UAAU,EAAE,OAAO,EAAE,UAAU;YAC/B,UAAU,EAAE,OAAO,EAAE,UAAU;SAChC,CAAC;QAEF,IAAI,QAAQ,CAAC;QACb,IAAI;YACF,QAAQ,GAAG,MAAM,IAAA,0BAAkB,EACjC,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,uBAAuB,aAAa,EACrE;gBACE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACP,cAAc,EAAE,kBAAkB;iBACnC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;aAC3B,EACD,2BAAkB,CACnB,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,mBAAmB,MAAM,CAAC,GAAG,CAAC,EAAE;aACxC,CAAC;SACH;QAED,IAAI;YACF,OAAO;gBACL,MAAM,EAAE,QAAQ,CAAC,IAAI,CAAC,OAAO;aAC9B,CAAC;SACH;QAAC,OAAO,GAAG,EAAE;YACZ,OAAO;gBACL,KAAK,EAAE,uBAAuB,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE;aAC9E,CAAC;SACH;IACH,CAAC;CACF;AArED,sCAqEC"}

package/dist/src/providers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAOrF,OAAO,KAAK,EACV,WAAW,EACX,cAAc,EACd,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EAClB,MAAM,SAAS,CAAC;AAEjB,wBAAsB,gBAAgB,CACpC,aAAa,EAAE,UAAU,GAAG,UAAU,EAAE,GAAG,iBAAiB,EAAE,GAAG,gBAAgB,EACjF,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,WAAW,EAAE,CAAC,CA6BxB;AAED,wBAAsB,eAAe,CACnC,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE,cAAc,EACxB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,WAAW,CAAC,CA8FtB;;;;;;;;;;AAED,wBAQE"}
+{"version":3,"file":"providers.d.ts","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,wBAAwB,EAAE,4BAA4B,EAAE,MAAM,oBAAoB,CAAC;AAC5F,OAAO,EAAE,2BAA2B,EAAE,MAAM,uBAAuB,CAAC;AACpE,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAQrF,OAAO,KAAK,EACV,WAAW,EACX,cAAc,EACd,gBAAgB,EAChB,UAAU,EACV,iBAAiB,EAClB,MAAM,SAAS,CAAC;AAEjB,wBAAsB,gBAAgB,CACpC,aAAa,EAAE,UAAU,GAAG,UAAU,EAAE,GAAG,iBAAiB,EAAE,GAAG,gBAAgB,EACjF,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,WAAW,EAAE,CAAC,CA6BxB;AAED,wBAAsB,eAAe,CACnC,YAAY,EAAE,MAAM,EACpB,OAAO,CAAC,EAAE,cAAc,EACxB,QAAQ,CAAC,EAAE,MAAM,GAChB,OAAO,CAAC,WAAW,CAAC,CAiGtB;;;;;;;;;;AAED,wBAQE"}

package/dist/src/providers.js

@@ -32,6 +32,7 @@ const openai_1 = require("./providers/openai");
 const anthropic_1 = require("./providers/anthropic");
 const replicate_1 = require("./providers/replicate");
 const localai_1 = require("./providers/localai");
+const llama_1 = require("./providers/llama");
 const scriptCompletion_1 = require("./providers/scriptCompletion");
 const azureopenai_1 = require("./providers/azureopenai");
 async function loadApiProviders(providerPaths, basePath) {
@@ -133,7 +134,11 @@ async function loadApiProvider(providerPath, context, basePath) {
         const modelName = options.slice(1).join(':');
         return new replicate_1.ReplicateProvider(modelName, undefined, context?.config);
     }
-    if (providerPath?.startsWith('localai:')) {
+    if (providerPath === 'llama' || providerPath.startsWith('llama:')) {
+        const modelName = providerPath.split(':')[1];
+        return new llama_1.LlamaProvider(modelName, context?.config);
+    }
+    else if (providerPath?.startsWith('localai:')) {
         const options = providerPath.split(':');
         const modelType = options[1];
         const modelName = options[2];

package/dist/src/providers.js.map

@@ -1 +1 @@
-{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AAExB,+CAA4F;AAC5F,qDAAoE;AACpE,qDAA0D;AAC1D,iDAAqF;AACrF,mEAAwE;AACxE,yDAGiC;AAU1B,KAAK,UAAU,gBAAgB,CACpC,aAAiF,EACjF,QAAiB;IAEjB,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;KACpE;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE;QAC9C,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,iBAAiB;gBAC3B,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;KACH;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YAClC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;gBAChC,OAAO,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;aACvD;iBAAM,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;gBACzC,OAAO;oBACL,EAAE,EAAE,GAAG,EAAE,CAAC,mBAAmB,GAAG,EAAE;oBAClC,OAAO,EAAE,QAAQ;iBAClB,CAAC;aACH;iBAAM;gBACL,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,OAAO,GAAG,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;gBACxC,OAAO,eAAe,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;aAC/C;QACH,CAAC,CAAC,CACH,CAAC;KACH;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAhCD,4CAgCC;AAEM,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,OAAwB,EACxB,QAAiB;IAEjB,IAAI,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE;QACrC,qBAAqB;QACrB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,2CAAwB,CAAC,UAAU,EAAE;YAC9C,EAAE,EAAE,QAAQ,UAAU,EAAE;YACxB,MAAM,EAAE,EAAE,QAAQ,EAAE;SACrB,CAAC,CAAC;KACJ;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QAC9C,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CACrC,SAAS,IAAI,eAAe,EAC5B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CACjC,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAChF;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC5E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,2BAA2B;QAC3B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,+CAAiC,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC1F;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,2CAA6B,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SACtF;aAAM;YACL,MAAM,IAAI,KAAK,CACb,oCAAoC,SAAS,gGAAgG,CAC9I,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,YAAY,EAAE;YAC9B,OAAO,IAAI,uCAA2B,CACpC,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,uCAA2B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACtF,OAAO,IAAI,uCAA2B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC/E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,yEAAyE,CACpH,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE7C,OAAO,IAAI,6BAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;KACrE;IAED,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QACxC,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,CAAC,CAAC;SACjD;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AAlGD,0CAkGC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,2BAA2B,EAA3B,uCAA2B;IAC3B,iBAAiB,EAAjB,6BAAiB;IACjB,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}
+{"version":3,"file":"providers.js","sourceRoot":"","sources":["../../src/providers.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAwB;AAExB,+CAA4F;AAC5F,qDAAoE;AACpE,qDAA0D;AAC1D,iDAAqF;AACrF,6CAAkD;AAClD,mEAAwE;AACxE,yDAGiC;AAU1B,KAAK,UAAU,gBAAgB,CACpC,aAAiF,EACjF,QAAiB;IAEjB,IAAI,OAAO,aAAa,KAAK,QAAQ,EAAE;QACrC,OAAO,CAAC,MAAM,eAAe,CAAC,aAAa,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC,CAAC;KACpE;SAAM,IAAI,OAAO,aAAa,KAAK,UAAU,EAAE;QAC9C,OAAO;YACL;gBACE,EAAE,EAAE,GAAG,EAAE,CAAC,iBAAiB;gBAC3B,OAAO,EAAE,aAAa;aACvB;SACF,CAAC;KACH;SAAM,IAAI,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE;QACvC,OAAO,OAAO,CAAC,GAAG,CAChB,aAAa,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE;YAClC,IAAI,OAAO,QAAQ,KAAK,QAAQ,EAAE;gBAChC,OAAO,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;aACvD;iBAAM,IAAI,OAAO,QAAQ,KAAK,UAAU,EAAE;gBACzC,OAAO;oBACL,EAAE,EAAE,GAAG,EAAE,CAAC,mBAAmB,GAAG,EAAE;oBAClC,OAAO,EAAE,QAAQ;iBAClB,CAAC;aACH;iBAAM;gBACL,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;gBACpC,MAAM,OAAO,GAAG,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC;gBACxC,OAAO,eAAe,CAAC,EAAE,EAAE,OAAO,EAAE,QAAQ,CAAC,CAAC;aAC/C;QACH,CAAC,CAAC,CACH,CAAC;KACH;IACD,MAAM,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC;AAC5C,CAAC;AAhCD,4CAgCC;AAEM,KAAK,UAAU,eAAe,CACnC,YAAoB,EACpB,OAAwB,EACxB,QAAiB;IAEjB,IAAI,YAAY,EAAE,UAAU,CAAC,OAAO,CAAC,EAAE;QACrC,qBAAqB;QACrB,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,OAAO,IAAI,2CAAwB,CAAC,UAAU,EAAE;YAC9C,EAAE,EAAE,QAAQ,UAAU,EAAE;YACxB,MAAM,EAAE,EAAE,QAAQ,EAAE;SACrB,CAAC,CAAC;KACJ;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,SAAS,CAAC,EAAE;QAC9C,qBAAqB;QACrB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,qCAA4B,CACrC,SAAS,IAAI,eAAe,EAC5B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,iCAAwB,CACjC,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,qCAA4B,CAAC,kBAAkB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAC9E,OAAO,IAAI,qCAA4B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAChF;aAAM,IAAI,iCAAwB,CAAC,wBAAwB,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YAChF,OAAO,IAAI,iCAAwB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC5E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,8BAA8B,SAAS,gGAAgG,CACxI,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,cAAc,CAAC,EAAE;QACnD,2BAA2B;QAC3B,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAElC,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,+CAAiC,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC1F;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,2CAA6B,CAAC,cAAc,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SACtF;aAAM;YACL,MAAM,IAAI,KAAK,CACb,oCAAoC,SAAS,gGAAgG,CAC9I,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,YAAY,EAAE;YAC9B,OAAO,IAAI,uCAA2B,CACpC,SAAS,IAAI,kBAAkB,EAC/B,SAAS,EACT,OAAO,EAAE,MAAM,CAChB,CAAC;SACH;aAAM,IAAI,uCAA2B,CAAC,2BAA2B,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE;YACtF,OAAO,IAAI,uCAA2B,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;SAC/E;aAAM;YACL,MAAM,IAAI,KAAK,CACb,iCAAiC,SAAS,yEAAyE,CACpH,CAAC;SACH;KACF;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,YAAY,CAAC,EAAE;QACjD,wBAAwB;QACxB,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE7C,OAAO,IAAI,6BAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;KACrE;IAED,IAAI,YAAY,KAAK,OAAO,IAAI,YAAY,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE;QACjE,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC7C,OAAO,IAAI,qBAAa,CAAC,SAAS,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;KACtD;SAAM,IAAI,YAAY,EAAE,UAAU,CAAC,UAAU,CAAC,EAAE;QAC/C,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACxC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAE7B,IAAI,SAAS,KAAK,MAAM,EAAE;YACxB,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;aAAM,IAAI,SAAS,KAAK,YAAY,EAAE;YACrC,OAAO,IAAI,mCAAyB,CAAC,SAAS,CAAC,CAAC;SACjD;aAAM;YACL,OAAO,IAAI,6BAAmB,CAAC,SAAS,CAAC,CAAC;SAC3C;KACF;IAED,qBAAqB;IACrB,MAAM,iBAAiB,GAAG,CAAC,yBAAa,cAAI,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,YAAY,CAAC,uCAAC,CAAC,CAAC,OAAO,CAAC;IACzF,OAAO,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;AACxC,CAAC;AArGD,0CAqGC;AAED,kBAAe;IACb,wBAAwB,EAAxB,iCAAwB;IACxB,4BAA4B,EAA5B,qCAA4B;IAC5B,2BAA2B,EAA3B,uCAA2B;IAC3B,iBAAiB,EAAjB,6BAAiB;IACjB,yBAAyB,EAAzB,mCAAyB;IACzB,mBAAmB,EAAnB,6BAAmB;IACnB,eAAe;CAChB,CAAC"}

package/dist/src/util.js

@@ -409,7 +409,7 @@ function writeLatestResults(results, config) {
             config,
             results,
         }, null, 2));
-        if (fs.existsSync(latestResultsPath) || fs.lstatSync(latestResultsPath).isSymbolicLink()) {
+        if (fs.existsSync(latestResultsPath) && fs.lstatSync(latestResultsPath).isSymbolicLink()) {
             fs.unlinkSync(latestResultsPath);
         }
         fs.symlinkSync(newResultsPath, latestResultsPath);

package/package.json

@@ -2,7 +2,7 @@
   "name": "promptfoo",
   "description": "LLM eval & testing toolkit",
   "author": "Ian Webster",
-  "version": "0.18.0",
+  "version": "0.18.1",
   "license": "MIT",
   "type": "commonjs",
   "main": "dist/src/index.js",

package/src/providers/llama.ts

@@ -0,0 +1,95 @@
+import { fetchJsonWithCache } from '../cache';
+import { REQUEST_TIMEOUT_MS } from './shared';
+
+import type { ApiProvider, ProviderResponse } from '../types.js';
+
+interface LlamaCompletionOptions {
+  n_predict?: number;
+  temperature?: number;
+  top_k?: number;
+  top_p?: number;
+  n_keep?: number;
+  stop?: string[];
+  repeat_penalty?: number;
+  repeat_last_n?: number;
+  penalize_nl?: boolean;
+  presence_penalty?: number;
+  frequency_penalty?: number;
+  mirostat?: boolean;
+  mirostat_tau?: number;
+  mirostat_eta?: number;
+  seed?: number;
+  ignore_eos?: boolean;
+  logit_bias?: Record<string, number>;
+}
+
+export class LlamaProvider implements ApiProvider {
+  modelName: string;
+  options?: LlamaCompletionOptions;
+
+  constructor(modelName: string, options?: LlamaCompletionOptions) {
+    this.modelName = modelName;
+    this.options = options;
+  }
+
+  id(): string {
+    return `llama:${this.modelName}`;
+  }
+
+  toString(): string {
+    return `[Llama Provider ${this.modelName}]`;
+  }
+
+  async callApi(prompt: string, options?: LlamaCompletionOptions): Promise<ProviderResponse> {
+    options = Object.assign({}, this.options, options);
+    const body = {
+      prompt,
+      n_predict: options?.n_predict || 512,
+      temperature: options?.temperature,
+      top_k: options?.top_k,
+      top_p: options?.top_p,
+      n_keep: options?.n_keep,
+      stop: options?.stop,
+      repeat_penalty: options?.repeat_penalty,
+      repeat_last_n: options?.repeat_last_n,
+      penalize_nl: options?.penalize_nl,
+      presence_penalty: options?.presence_penalty,
+      frequency_penalty: options?.frequency_penalty,
+      mirostat: options?.mirostat,
+      mirostat_tau: options?.mirostat_tau,
+      mirostat_eta: options?.mirostat_eta,
+      seed: options?.seed,
+      ignore_eos: options?.ignore_eos,
+      logit_bias: options?.logit_bias,
+    };
+
+    let response;
+    try {
+      response = await fetchJsonWithCache(
+        `${process.env.LLAMA_BASE_URL || 'http://localhost:8080'}/completion`,
+        {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify(body),
+        },
+        REQUEST_TIMEOUT_MS,
+      );
+    } catch (err) {
+      return {
+        error: `API call error: ${String(err)}`,
+      };
+    }
+
+    try {
+      return {
+        output: response.data.content,
+      };
+    } catch (err) {
+      return {
+        error: `API response error: ${String(err)}: ${JSON.stringify(response.data)}`,
+      };
+    }
+  }
+}

package/src/providers.ts

@@ -4,6 +4,7 @@ import { OpenAiCompletionProvider, OpenAiChatCompletionProvider } from './provid
 import { AnthropicCompletionProvider } from './providers/anthropic';
 import { ReplicateProvider } from './providers/replicate';
 import { LocalAiCompletionProvider, LocalAiChatProvider } from './providers/localai';
+import { LlamaProvider } from './providers/llama';
 import { ScriptCompletionProvider } from './providers/scriptCompletion';
 import {
   AzureOpenAiChatCompletionProvider,
@@ -133,7 +134,10 @@ export async function loadApiProvider(
     return new ReplicateProvider(modelName, undefined, context?.config);
   }
 
-  if (providerPath?.startsWith('localai:')) {
+  if (providerPath === 'llama' || providerPath.startsWith('llama:')) {
+    const modelName = providerPath.split(':')[1];
+    return new LlamaProvider(modelName, context?.config);
+  } else if (providerPath?.startsWith('localai:')) {
     const options = providerPath.split(':');
     const modelType = options[1];
     const modelName = options[2];

package/src/util.ts

@@ -456,7 +456,7 @@ export function writeLatestResults(results: EvaluateSummary, config: Partial<Uni
         2,
       ),
     );
-    if (fs.existsSync(latestResultsPath) || fs.lstatSync(latestResultsPath).isSymbolicLink()) {
+    if (fs.existsSync(latestResultsPath) && fs.lstatSync(latestResultsPath).isSymbolicLink()) {
       fs.unlinkSync(latestResultsPath);
     }
     fs.symlinkSync(newResultsPath, latestResultsPath);