promptfoo 0.72.1 → 0.73.0

package/dist/test/prompts.utils.test.js

@@ -0,0 +1,123 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const utils_1 = require("../src/prompts/utils");
+jest.mock('fs', () => ({
+    statSync: jest.fn(jest.requireActual('fs').statSync),
+}));
+jest.mock('glob', () => ({
+    globSync: jest.fn(jest.requireActual('glob').globSync),
+}));
+describe('maybeFilePath', () => {
+    it('should return true for valid file paths', () => {
+        expect((0, utils_1.maybeFilePath)('C:\\path\\to\\file.txt')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('file.*')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.ext')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('path/to/file.txt')).toBe(true);
+    });
+    it('should return false for strings with new lines', () => {
+        expect((0, utils_1.maybeFilePath)('path/to\nfile.txt')).toBe(false);
+        expect((0, utils_1.maybeFilePath)('file\nname.ext')).toBe(false);
+    });
+    it('should return false for strings with "portkey://"', () => {
+        expect((0, utils_1.maybeFilePath)('portkey://path/to/file')).toBe(false);
+    });
+    it('should return false for strings with "langfuse://"', () => {
+        expect((0, utils_1.maybeFilePath)('langfuse://path/to/file')).toBe(false);
+    });
+    it('should return false for strings without file path indicators', () => {
+        expect((0, utils_1.maybeFilePath)('anotherstring')).toBe(false);
+        expect((0, utils_1.maybeFilePath)('justastring')).toBe(false);
+        expect((0, utils_1.maybeFilePath)('stringwith.dotbutnotfile')).toBe(false);
+    });
+    it('should return true for strings with file:// prefix', () => {
+        expect((0, utils_1.maybeFilePath)('file://path/to/file.txt')).toBe(true);
+    });
+    it('should return true for strings with wildcard character', () => {
+        expect((0, utils_1.maybeFilePath)('*.txt')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('path/to/*.txt')).toBe(true);
+    });
+    it('should return true for strings with file extension at the third or fourth last position', () => {
+        expect((0, utils_1.maybeFilePath)('file.ext')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('file.name.ext')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.e')).toBe(false);
+        expect((0, utils_1.maybeFilePath)('filename.ex')).toBe(true);
+    });
+    it('should work for files that end with specific allowed extensions', () => {
+        expect((0, utils_1.maybeFilePath)('filename.cjs')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.js')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.js:functionName')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.json')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.jsonl')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.mjs')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.py')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.py:functionName')).toBe(true);
+        expect((0, utils_1.maybeFilePath)('filename.txt')).toBe(true);
+    });
+    // Additional tests
+    it('should return false for empty strings', () => {
+        expect((0, utils_1.maybeFilePath)('')).toBe(false);
+    });
+    it('should return false for whitespace strings', () => {
+        expect((0, utils_1.maybeFilePath)('   ')).toBe(false);
+        expect((0, utils_1.maybeFilePath)('\t')).toBe(false);
+    });
+    it('should return false for non-string inputs', () => {
+        expect(() => (0, utils_1.maybeFilePath)(123)).toThrow('Invalid input: 123');
+        expect(() => (0, utils_1.maybeFilePath)({})).toThrow('Invalid input: {}');
+        expect(() => (0, utils_1.maybeFilePath)([])).toThrow('Invalid input: []');
+    });
+    it('should return false for strings with invalid and valid indicators mixed', () => {
+        expect((0, utils_1.maybeFilePath)('file://path/to\nfile.txt')).toBe(false);
+        expect((0, utils_1.maybeFilePath)('path/to/file.txtportkey://')).toBe(false);
+    });
+    it('should return true for very long valid file paths', () => {
+        const longPath = 'a/'.repeat(100) + 'file.txt';
+        expect((0, utils_1.maybeFilePath)(longPath)).toBe(true);
+    });
+    it('should return false for very long invalid file paths', () => {
+        const longInvalidPath = 'a/'.repeat(100) + 'file\n.txt';
+        expect((0, utils_1.maybeFilePath)(longInvalidPath)).toBe(false);
+    });
+    it('should return false for strings ending with a dot', () => {
+        expect((0, utils_1.maybeFilePath)('Write a tweet about {{topic}}.')).toBe(false);
+    });
+});
+describe('normalizeInput', () => {
+    it('rejects invalid input types', () => {
+        expect(() => (0, utils_1.normalizeInput)(null)).toThrow('Invalid input prompt: null');
+        expect(() => (0, utils_1.normalizeInput)(undefined)).toThrow('Invalid input prompt: undefined');
+        expect(() => (0, utils_1.normalizeInput)(1)).toThrow('Invalid input prompt: 1');
+        expect(() => (0, utils_1.normalizeInput)(true)).toThrow('Invalid input prompt: true');
+        expect(() => (0, utils_1.normalizeInput)(false)).toThrow('Invalid input prompt: false');
+    });
+    it('rejects empty inputs', () => {
+        expect(() => (0, utils_1.normalizeInput)([])).toThrow('Invalid input prompt: []');
+        expect(() => (0, utils_1.normalizeInput)({})).toThrow('Invalid input prompt: {}');
+        expect(() => (0, utils_1.normalizeInput)('')).toThrow('Invalid input prompt: ""');
+    });
+    it('returns array with single string when input is a non-empty string', () => {
+        expect((0, utils_1.normalizeInput)('valid string')).toEqual([{ raw: 'valid string' }]);
+    });
+    it('returns input array when input is a non-empty array', () => {
+        const inputArray = ['prompt1', { raw: 'prompt2' }];
+        expect((0, utils_1.normalizeInput)(inputArray)).toEqual([{ raw: 'prompt1' }, { raw: 'prompt2' }]);
+    });
+    // NOTE: Legacy mode. This is deprecated and will be removed in a future version.
+    it('returns array of prompts when input is an object', () => {
+        const inputObject = {
+            'prompts1.txt': 'label A',
+            'prompts2.txt': 'label B',
+        };
+        expect((0, utils_1.normalizeInput)(inputObject)).toEqual([
+            {
+                label: 'label A',
+                raw: 'prompts1.txt',
+            },
+            {
+                label: 'label B',
+                raw: 'prompts2.txt',
+            },
+        ]);
+    });
+});
+//# sourceMappingURL=prompts.utils.test.js.map

package/dist/test/prompts.utils.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompts.utils.test.js","sourceRoot":"","sources":["../../test/prompts.utils.test.ts"],"names":[],"mappings":";;AAAA,gDAAqE;AAErE,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,GAAG,EAAE,CAAC,CAAC;IACrB,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC;CACrD,CAAC,CAAC,CAAC;AAEJ,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC;IACvB,QAAQ,EAAE,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC;CACvD,CAAC,CAAC,CAAC;AAEJ,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,MAAM,CAAC,IAAA,qBAAa,EAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,MAAM,CAAC,IAAA,qBAAa,EAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3C,MAAM,CAAC,IAAA,qBAAa,EAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,CAAC,IAAA,qBAAa,EAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,CAAC,IAAA,qBAAa,EAAC,mBAAmB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACvD,MAAM,CAAC,IAAA,qBAAa,EAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,IAAA,qBAAa,EAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,CAAC,IAAA,qBAAa,EAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,MAAM,CAAC,IAAA,qBAAa,EAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACnD,MAAM,CAAC,IAAA,qBAAa,EAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,CAAC,IAAA,qBAAa,EAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,CAAC,IAAA,qBAAa,EAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,CAAC,IAAA,qBAAa,EAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1C,MAAM,CAAC,IAAA,qBAAa,EAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yFAAyF,EAAE,GAAG,EAAE;QACjG,MAAM,CAAC,IAAA,qBAAa,EAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7C,MAAM,CAAC,IAAA,qBAAa,EAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,CAAC,IAAA,qBAAa,EAAC,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChD,MAAM,CAAC,IAAA,qBAAa,EAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACzE,MAAM,CAAC,IAAA,qBAAa,EAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,CAAC,IAAA,qBAAa,EAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,IAAA,qBAAa,EAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,CAAC,IAAA,qBAAa,EAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,CAAC,IAAA,qBAAa,EAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,CAAC,IAAA,qBAAa,EAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjD,MAAM,CAAC,IAAA,qBAAa,EAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,IAAA,qBAAa,EAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,MAAM,CAAC,IAAA,qBAAa,EAAC,cAAc,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,mBAAmB;IACnB,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,CAAC,IAAA,qBAAa,EAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,CAAC,IAAA,qBAAa,EAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzC,MAAM,CAAC,IAAA,qBAAa,EAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,GAAG,EAAE;QACnD,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,qBAAa,EAAC,GAAY,CAAC,CAAC,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QACxE,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,qBAAa,EAAC,EAAW,CAAC,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACtE,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,qBAAa,EAAC,EAAW,CAAC,CAAC,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,GAAG,EAAE;QACjF,MAAM,CAAC,IAAA,qBAAa,EAAC,0BAA0B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9D,MAAM,CAAC,IAAA,qBAAa,EAAC,4BAA4B,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC;QAC/C,MAAM,CAAC,IAAA,qBAAa,EAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,eAAe,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,YAAY,CAAC;QACxD,MAAM,CAAC,IAAA,qBAAa,EAAC,eAAe,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,CAAC,IAAA,qBAAa,EAAC,gCAAgC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACtE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,6BAA6B,EAAE,GAAG,EAAE;QACrC,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,sBAAc,EAAC,IAAW,CAAC,CAAC,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;QAChF,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,sBAAc,EAAC,SAAgB,CAAC,CAAC,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;QAC1F,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,sBAAc,EAAC,CAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC;QAC1E,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,sBAAc,EAAC,IAAW,CAAC,CAAC,CAAC,OAAO,CAAC,4BAA4B,CAAC,CAAC;QAChF,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,sBAAc,EAAC,KAAY,CAAC,CAAC,CAAC,OAAO,CAAC,6BAA6B,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sBAAsB,EAAE,GAAG,EAAE;QAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,sBAAc,EAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;QACrE,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,sBAAc,EAAC,EAAS,CAAC,CAAC,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;QAC5E,MAAM,CAAC,GAAG,EAAE,CAAC,IAAA,sBAAc,EAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;IACvE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,mEAAmE,EAAE,GAAG,EAAE;QAC3E,MAAM,CAAC,IAAA,sBAAc,EAAC,cAAc,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qDAAqD,EAAE,GAAG,EAAE;QAC7D,MAAM,UAAU,GAAG,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC;QACnD,MAAM,CAAC,IAAA,sBAAc,EAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;IACvF,CAAC,CAAC,CAAC;IAEH,iFAAiF;IACjF,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,WAAW,GAAG;YAClB,cAAc,EAAE,SAAS;YACzB,cAAc,EAAE,SAAS;SAC1B,CAAC;QACF,MAAM,CAAC,IAAA,sBAAc,EAAC,WAAW,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1C;gBACE,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,cAAc;aACpB;YACD;gBACE,KAAK,EAAE,SAAS;gBAChB,GAAG,EAAE,cAAc;aACpB;SACF,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/providers.anthropic.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=providers.anthropic.test.d.ts.map

package/dist/test/providers.anthropic.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"providers.anthropic.test.d.ts","sourceRoot":"","sources":["../../test/providers.anthropic.test.ts"],"names":[],"mappings":""}

package/dist/test/providers.anthropic.test.js

@@ -0,0 +1,520 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const dedent_1 = __importDefault(require("dedent"));
+const cache_1 = require("../src/cache");
+const anthropic_1 = require("../src/providers/anthropic");
+jest.mock('proxy-agent', () => ({
+    ProxyAgent: jest.fn().mockImplementation(() => ({})),
+}));
+describe('Anthropic', () => {
+    afterEach(async () => {
+        jest.clearAllMocks();
+        await (0, cache_1.clearCache)();
+    });
+    describe('AnthropicMessagesProvider callApi', () => {
+        const tools = [
+            {
+                name: 'get_weather',
+                description: 'Get the current weather in a given location',
+                input_schema: {
+                    type: 'object',
+                    properties: {
+                        location: {
+                            type: 'string',
+                            description: 'The city and state, e.g. San Francisco, CA',
+                        },
+                        unit: {
+                            type: 'string',
+                            enum: ['celsius', 'fahrenheit'],
+                        },
+                    },
+                    required: ['location'],
+                },
+            },
+        ];
+        const provider = new anthropic_1.AnthropicMessagesProvider('claude-3-opus-20240229', {
+            config: { tools },
+        });
+        it('should use cache by default for ToolUse requests', async () => {
+            jest
+                .spyOn(provider.anthropic.messages, 'create')
+                .mockImplementation()
+                .mockResolvedValue({
+                content: [
+                    {
+                        type: 'text',
+                        text: '<thinking>I need to use the get_weather, and the user wants SF, which is likely San Francisco, CA.</thinking>',
+                    },
+                    {
+                        type: 'tool_use',
+                        id: 'toolu_01A09q90qw90lq917835lq9',
+                        name: 'get_weather',
+                        input: { location: 'San Francisco, CA', unit: 'celsius' },
+                    },
+                ],
+            });
+            const result = await provider.callApi('What is the forecast in San Francisco?');
+            expect(provider.anthropic.messages.create).toHaveBeenCalledTimes(1);
+            expect(provider.anthropic.messages.create).toHaveBeenNthCalledWith(1, {
+                model: 'claude-3-opus-20240229',
+                max_tokens: 1024,
+                messages: [
+                    {
+                        role: 'user',
+                        content: [
+                            {
+                                text: 'What is the forecast in San Francisco?',
+                                type: 'text',
+                            },
+                        ],
+                    },
+                ],
+                tools,
+                temperature: 0,
+                stream: false,
+            });
+            expect(result).toMatchObject({
+                cost: undefined,
+                output: (0, dedent_1.default) `<thinking>I need to use the get_weather, and the user wants SF, which is likely San Francisco, CA.</thinking>
+
+          {"type":"tool_use","id":"toolu_01A09q90qw90lq917835lq9","name":"get_weather","input":{"location":"San Francisco, CA","unit":"celsius"}}`,
+                tokenUsage: {},
+            });
+            const resultFromCache = await provider.callApi('What is the forecast in San Francisco?');
+            expect(provider.anthropic.messages.create).toHaveBeenCalledTimes(1);
+            expect(result).toMatchObject(resultFromCache);
+        });
+        it('should pass the tool choice if specified', async () => {
+            const toolChoice = {
+                name: 'get_weather',
+                type: 'tool',
+            };
+            provider.config.tool_choice = toolChoice;
+            jest
+                .spyOn(provider.anthropic.messages, 'create')
+                .mockImplementation()
+                .mockResolvedValue({
+                content: [
+                    {
+                        type: 'text',
+                        text: '<thinking>I need to use the get_weather, and the user wants SF, which is likely San Francisco, CA.</thinking>',
+                    },
+                    {
+                        type: 'tool_use',
+                        id: 'toolu_01A09q90qw90lq917835lq9',
+                        name: 'get_weather',
+                        input: { location: 'San Francisco, CA', unit: 'celsius' },
+                    },
+                ],
+            });
+            const result = await provider.callApi('What is the forecast in San Francisco?');
+            expect(provider.anthropic.messages.create).toHaveBeenCalledTimes(1);
+            expect(provider.anthropic.messages.create).toHaveBeenNthCalledWith(1, {
+                model: 'claude-3-opus-20240229',
+                max_tokens: 1024,
+                messages: [
+                    {
+                        role: 'user',
+                        content: [
+                            {
+                                text: 'What is the forecast in San Francisco?',
+                                type: 'text',
+                            },
+                        ],
+                    },
+                ],
+                tools,
+                tool_choice: toolChoice,
+                temperature: 0,
+                stream: false,
+            });
+            provider.config.tool_choice = undefined;
+        });
+        it('should not use cache if caching is disabled for ToolUse requests', async () => {
+            jest
+                .spyOn(provider.anthropic.messages, 'create')
+                .mockImplementation()
+                .mockResolvedValue({
+                content: [
+                    {
+                        type: 'text',
+                        text: '<thinking>I need to use the get_weather, and the user wants SF, which is likely San Francisco, CA.</thinking>',
+                    },
+                    {
+                        type: 'tool_use',
+                        id: 'toolu_01A09q90qw90lq917835lq9',
+                        name: 'get_weather',
+                        input: { location: 'San Francisco, CA', unit: 'celsius' },
+                    },
+                ],
+            });
+            (0, cache_1.disableCache)();
+            const result = await provider.callApi('What is the forecast in San Francisco?');
+            expect(provider.anthropic.messages.create).toHaveBeenCalledTimes(1);
+            expect(result).toMatchObject({
+                output: (0, dedent_1.default) `<thinking>I need to use the get_weather, and the user wants SF, which is likely San Francisco, CA.</thinking>
+
+          {"type":"tool_use","id":"toolu_01A09q90qw90lq917835lq9","name":"get_weather","input":{"location":"San Francisco, CA","unit":"celsius"}}`,
+                tokenUsage: {},
+            });
+            await provider.callApi('What is the forecast in San Francisco?');
+            expect(provider.anthropic.messages.create).toHaveBeenCalledTimes(2);
+            (0, cache_1.enableCache)();
+        });
+        it('should return cached response for legacy caching behavior', async () => {
+            jest
+                .spyOn(provider.anthropic.messages, 'create')
+                .mockImplementation()
+                .mockResolvedValue({
+                content: [],
+            });
+            (0, cache_1.getCache)().set('anthropic:{"model":"claude-3-opus-20240229","max_tokens":1024,"messages":[{"role":"user","content":[{"type":"text","text":"What is the forecast in San Francisco?"}]}],"stream":false,"temperature":0,"tools":[{"name":"get_weather","description":"Get the current weather in a given location","input_schema":{"type":"object","properties":{"location":{"type":"string","description":"The city and state, e.g. San Francisco, CA"},"unit":{"type":"string","enum":["celsius","fahrenheit"]}},"required":["location"]}}]}', 'Test output');
+            const result = await provider.callApi('What is the forecast in San Francisco?');
+            expect(result).toMatchObject({
+                output: 'Test output',
+                tokenUsage: {},
+            });
+            expect(provider.anthropic.messages.create).toHaveBeenCalledTimes(0);
+        });
+        it('should handle API call error', async () => {
+            const provider = new anthropic_1.AnthropicMessagesProvider('claude-3-opus-20240229');
+            jest
+                .spyOn(provider.anthropic.messages, 'create')
+                .mockImplementation()
+                .mockRejectedValue(new Error('API call failed'));
+            const result = await provider.callApi('What is the forecast in San Francisco?');
+            expect(result).toMatchObject({
+                error: 'API call error: Error: API call failed',
+            });
+        });
+        it('should return token usage and cost', async () => {
+            const provider = new anthropic_1.AnthropicMessagesProvider('claude-3-opus-20240229', {
+                config: { max_tokens: 100, temperature: 0.5, cost: 0.015 },
+            });
+            jest
+                .spyOn(provider.anthropic.messages, 'create')
+                .mockImplementation()
+                .mockResolvedValue({
+                content: [{ type: 'text', text: 'Test output' }],
+                usage: { input_tokens: 50, output_tokens: 50 },
+            });
+            const result = await provider.callApi('What is the forecast in San Francisco?');
+            expect(result).toMatchObject({
+                output: 'Test output',
+                tokenUsage: { total: 100, prompt: 50, completion: 50 },
+                cost: 1.5,
+            });
+        });
+    });
+    describe('AnthropicLlmRubricProvider', () => {
+        let provider;
+        beforeEach(() => {
+            provider = new anthropic_1.AnthropicLlmRubricProvider('claude-3-5-sonnet-20240620');
+        });
+        it('should initialize with forced tool configuration', () => {
+            expect(provider.modelName).toBe('claude-3-5-sonnet-20240620');
+            expect(provider.config.tool_choice).toEqual({ type: 'tool', name: 'grade_output' });
+        });
+        it('should call API and parse the result correctly', async () => {
+            const mockApiResponse = {
+                output: JSON.stringify({
+                    type: 'tool_use',
+                    id: 'test-id',
+                    name: 'grade_output',
+                    input: {
+                        pass: true,
+                        score: 0.85,
+                        reason: 'The output meets the criteria.',
+                    },
+                }),
+            };
+            jest.spyOn(anthropic_1.AnthropicMessagesProvider.prototype, 'callApi').mockResolvedValue(mockApiResponse);
+            const result = await provider.callApi('Test prompt');
+            expect(result).toEqual({
+                output: {
+                    pass: true,
+                    score: 0.85,
+                    reason: 'The output meets the criteria.',
+                },
+            });
+        });
+        it('should handle non-string API response', async () => {
+            const mockApiResponse = {
+                output: { confession: 'I am not a string' },
+            };
+            jest.spyOn(anthropic_1.AnthropicMessagesProvider.prototype, 'callApi').mockResolvedValue(mockApiResponse);
+            const result = await provider.callApi('Test prompt');
+            expect(result.error).toContain('Anthropic LLM rubric grader - malformed non-string output');
+        });
+        it('should handle malformed API response', async () => {
+            const mockApiResponse = {
+                output: 'Invalid JSON',
+            };
+            jest.spyOn(anthropic_1.AnthropicMessagesProvider.prototype, 'callApi').mockResolvedValue(mockApiResponse);
+            const result = await provider.callApi('Test prompt');
+            expect(result.error).toContain('Anthropic LLM rubric grader - invalid JSON');
+        });
+        it('should handle API errors', async () => {
+            const mockError = new Error('API Error');
+            jest.spyOn(anthropic_1.AnthropicMessagesProvider.prototype, 'callApi').mockRejectedValue(mockError);
+            await expect(provider.callApi('Test prompt')).rejects.toThrow('API Error');
+        });
+    });
+    describe('AnthropicCompletionProvider callApi', () => {
+        it('should return output for default behavior', async () => {
+            const provider = new anthropic_1.AnthropicCompletionProvider('claude-1');
+            jest.spyOn(provider.anthropic.completions, 'create').mockImplementation().mockResolvedValue({
+                id: 'test-id',
+                model: 'claude-1',
+                stop_reason: 'stop_sequence',
+                type: 'completion',
+                completion: 'Test output',
+            });
+            const result = await provider.callApi('Test prompt');
+            expect(provider.anthropic.completions.create).toHaveBeenCalledTimes(1);
+            expect(result).toMatchObject({
+                output: 'Test output',
+                tokenUsage: {},
+            });
+        });
+        it('should return cached output with caching enabled', async () => {
+            const provider = new anthropic_1.AnthropicCompletionProvider('claude-1');
+            jest.spyOn(provider.anthropic.completions, 'create').mockImplementation().mockResolvedValue({
+                id: 'test-id',
+                model: 'claude-1',
+                stop_reason: 'stop_sequence',
+                type: 'completion',
+                completion: 'Test output',
+            });
+            const result = await provider.callApi('Test prompt');
+            expect(provider.anthropic.completions.create).toHaveBeenCalledTimes(1);
+            expect(result).toMatchObject({
+                output: 'Test output',
+                tokenUsage: {},
+            });
+            jest.mocked(provider.anthropic.completions.create).mockClear();
+            const cachedResult = await provider.callApi('Test prompt');
+            expect(provider.anthropic.completions.create).toHaveBeenCalledTimes(0);
+            expect(cachedResult).toMatchObject({
+                output: 'Test output',
+                tokenUsage: {},
+            });
+        });
+        it('should return fresh output with caching disabled', async () => {
+            const provider = new anthropic_1.AnthropicCompletionProvider('claude-1');
+            jest.spyOn(provider.anthropic.completions, 'create').mockImplementation().mockResolvedValue({
+                id: 'test-id',
+                model: 'claude-1',
+                stop_reason: 'stop_sequence',
+                type: 'completion',
+                completion: 'Test output',
+            });
+            const result = await provider.callApi('Test prompt');
+            expect(provider.anthropic.completions.create).toHaveBeenCalledTimes(1);
+            expect(result).toMatchObject({
+                output: 'Test output',
+                tokenUsage: {},
+            });
+            jest.mocked(provider.anthropic.completions.create).mockClear();
+            (0, cache_1.disableCache)();
+            const freshResult = await provider.callApi('Test prompt');
+            expect(provider.anthropic.completions.create).toHaveBeenCalledTimes(1);
+            expect(freshResult).toMatchObject({
+                output: 'Test output',
+                tokenUsage: {},
+            });
+        });
+        it('should handle API call error', async () => {
+            const provider = new anthropic_1.AnthropicCompletionProvider('claude-1');
+            jest
+                .spyOn(provider.anthropic.completions, 'create')
+                .mockImplementation()
+                .mockRejectedValue(new Error('API call failed'));
+            const result = await provider.callApi('Test prompt');
+            expect(result).toMatchObject({
+                error: 'API call error: Error: API call failed',
+            });
+        });
+    });
+    describe('calculateCost', () => {
+        it('should calculate cost for valid input and output tokens', () => {
+            const cost = (0, anthropic_1.calculateCost)('claude-3-opus-20240229', { cost: 0.015 }, 100, 200);
+            expect(cost).toBe(4.5); // (0.015 * 100) + (0.075 * 200)
+        });
+        it('should return undefined for missing model', () => {
+            const cost = (0, anthropic_1.calculateCost)('non-existent-model', { cost: 0.015 });
+            expect(cost).toBeUndefined();
+        });
+        it('should return undefined for missing tokens', () => {
+            const cost = (0, anthropic_1.calculateCost)('claude-3-opus-20240229', { cost: 0.015 });
+            expect(cost).toBeUndefined();
+        });
+    });
+    describe('outputFromMessage', () => {
+        it('should return an empty string for empty content array', () => {
+            const message = {
+                content: [],
+                id: '',
+                model: '',
+                role: 'assistant',
+                stop_reason: null,
+                stop_sequence: null,
+                type: 'message',
+                usage: {
+                    input_tokens: 0,
+                    output_tokens: 0,
+                },
+            };
+            const result = (0, anthropic_1.outputFromMessage)(message);
+            expect(result).toBe('');
+        });
+        it('should return text from a single text block', () => {
+            const message = {
+                content: [{ type: 'text', text: 'Hello' }],
+                id: '',
+                model: '',
+                role: 'assistant',
+                stop_reason: null,
+                stop_sequence: null,
+                type: 'message',
+                usage: {
+                    input_tokens: 0,
+                    output_tokens: 0,
+                },
+            };
+            const result = (0, anthropic_1.outputFromMessage)(message);
+            expect(result).toBe('Hello');
+        });
+        it('should concatenate text blocks without tool_use blocks', () => {
+            const message = {
+                content: [
+                    { type: 'text', text: 'Hello' },
+                    { type: 'text', text: 'World' },
+                ],
+                id: '',
+                model: '',
+                role: 'assistant',
+                stop_reason: null,
+                stop_sequence: null,
+                type: 'message',
+                usage: {
+                    input_tokens: 0,
+                    output_tokens: 0,
+                },
+            };
+            const result = (0, anthropic_1.outputFromMessage)(message);
+            expect(result).toBe('Hello\n\nWorld');
+        });
+        it('should handle content with tool_use blocks', () => {
+            const message = {
+                content: [
+                    {
+                        type: 'tool_use',
+                        id: 'tool1',
+                        name: 'get_weather',
+                        input: { location: 'San Francisco, CA' },
+                    },
+                    {
+                        type: 'tool_use',
+                        id: 'tool2',
+                        name: 'get_time',
+                        input: { location: 'New York, NY' },
+                    },
+                ],
+                id: '',
+                model: '',
+                role: 'assistant',
+                stop_reason: null,
+                stop_sequence: null,
+                type: 'message',
+                usage: {
+                    input_tokens: 0,
+                    output_tokens: 0,
+                },
+            };
+            const result = (0, anthropic_1.outputFromMessage)(message);
+            expect(result).toBe('{"type":"tool_use","id":"tool1","name":"get_weather","input":{"location":"San Francisco, CA"}}\n\n{"type":"tool_use","id":"tool2","name":"get_time","input":{"location":"New York, NY"}}');
+        });
+        it('should concatenate text and tool_use blocks as JSON strings', () => {
+            const message = {
+                content: [
+                    { type: 'text', text: 'Hello' },
+                    {
+                        type: 'tool_use',
+                        id: 'tool1',
+                        name: 'get_weather',
+                        input: { location: 'San Francisco, CA' },
+                    },
+                    { type: 'text', text: 'World' },
+                ],
+                id: '',
+                model: '',
+                role: 'assistant',
+                stop_reason: null,
+                stop_sequence: null,
+                type: 'message',
+                usage: {
+                    input_tokens: 0,
+                    output_tokens: 0,
+                },
+            };
+            const result = (0, anthropic_1.outputFromMessage)(message);
+            expect(result).toBe('Hello\n\n{"type":"tool_use","id":"tool1","name":"get_weather","input":{"location":"San Francisco, CA"}}\n\nWorld');
+        });
+    });
+    describe('parseMessages', () => {
+        it('should parse messages with user and assistant roles', () => {
+            const inputMessages = (0, dedent_1.default) `user: What is the weather?
+          assistant: The weather is sunny.`;
+            const { system, extractedMessages } = (0, anthropic_1.parseMessages)(inputMessages);
+            expect(system).toBeUndefined();
+            expect(extractedMessages).toMatchObject([
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: (0, dedent_1.default) `user: What is the weather?
+                    assistant: The weather is sunny.`,
+                        },
+                    ],
+                },
+            ]);
+        });
+        it('should handle system messages', () => {
+            const inputMessages = (0, dedent_1.default) `system: This is a system message.
+        user: What is the weather?
+        assistant: The weather is sunny.`;
+            const { system, extractedMessages } = (0, anthropic_1.parseMessages)(inputMessages);
+            expect(system).toBeUndefined();
+            expect(extractedMessages).toMatchObject([
+                {
+                    role: 'user',
+                    content: [
+                        {
+                            type: 'text',
+                            text: (0, dedent_1.default) `system: This is a system message.
+              user: What is the weather?
+              assistant: The weather is sunny.`,
+                        },
+                    ],
+                },
+            ]);
+        });
+        it('should handle empty input', () => {
+            const inputMessages = '';
+            const { system, extractedMessages } = (0, anthropic_1.parseMessages)(inputMessages);
+            expect(system).toBeUndefined();
+            expect(extractedMessages).toMatchObject([
+                {
+                    role: 'user',
+                    content: [{ type: 'text', text: '' }],
+                },
+            ]);
+        });
+    });
+});
+//# sourceMappingURL=providers.anthropic.test.js.map