promptfoo 0.91.3 → 0.92.1

package/dist/test/server/share.test.js

@@ -0,0 +1,36 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const supertest_1 = __importDefault(require("supertest"));
+const migrate_1 = require("../../src/migrate");
+const eval_1 = __importDefault(require("../../src/models/eval"));
+const server_1 = require("../../src/server/server");
+const v3evalToShare_json_1 = __importDefault(require("./v3evalToShare.json"));
+const v4evalToShare_json_1 = __importDefault(require("./v4evalToShare.json"));
+describe('share', () => {
+    const app = (0, server_1.createApp)();
+    beforeAll(async () => {
+        await (0, migrate_1.runDbMigrations)();
+    });
+    it('should accept a version 3 results file', async () => {
+        const res = await (0, supertest_1.default)(app).post('/api/eval').send(v3evalToShare_json_1.default);
+        expect(res.status).toBe(200);
+        const eval_ = await eval_1.default.findById(res.body.id);
+        expect(eval_).not.toBeNull();
+        expect(eval_?.version()).toBe(3);
+        const results = await eval_?.getResults();
+        expect(results).toHaveLength(8);
+    });
+    it('should accept a new eval', async () => {
+        const res = await (0, supertest_1.default)(app).post('/api/eval').send(v4evalToShare_json_1.default);
+        expect(res.status).toBe(200);
+        const eval_ = await eval_1.default.findById(res.body.id);
+        expect(eval_).not.toBeNull();
+        expect(eval_?.version()).toBe(4);
+        const results = await eval_?.getResults();
+        expect(results).toHaveLength(8);
+    });
+});
+//# sourceMappingURL=share.test.js.map

package/dist/test/server/share.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"share.test.js","sourceRoot":"","sources":["../../../test/server/share.test.ts"],"names":[],"mappings":";;;;;AAAA,0DAAgC;AAChC,+CAAoD;AACpD,iEAAyC;AACzC,oDAAoD;AACpD,8EAA8C;AAC9C,8EAA8C;AAE9C,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE;IACrB,MAAM,GAAG,GAAG,IAAA,kBAAS,GAAE,CAAC;IACxB,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,MAAM,IAAA,yBAAe,GAAE,CAAC;IAC1B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,KAAK,IAAI,EAAE;QACtD,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAO,EAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,4BAAU,CAAC,CAAC;QAClE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,cAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAY,CAAC,CAAC;QACzD,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,KAAK,EAAE,UAAU,EAAE,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;QACxC,MAAM,GAAG,GAAG,MAAM,IAAA,mBAAO,EAAC,GAAG,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,4BAAU,CAAC,CAAC;QAClE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC7B,MAAM,KAAK,GAAG,MAAM,cAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,EAAY,CAAC,CAAC;QACzD,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC7B,MAAM,CAAC,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,KAAK,EAAE,UAAU,EAAE,CAAC;QAC1C,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/test/server/v3evalToShare.json

@@ -0,0 +1,507 @@
+{
+    "data": {
+        "version": 3,
+        "createdAt": "2024-10-02T16:30:21.627Z",
+        "author": "steve@promptfoo.dev",
+        "results": {
+            "version": 2,
+            "timestamp": "2024-10-02T16:29:47.070Z",
+            "results": [
+                {
+                    "provider": { "id": "openai:gpt-4o-mini", "label": "" },
+                    "prompt": {
+                        "raw": "Translate this to conversational French: I'm hungry",
+                        "label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}"
+                    },
+                    "vars": { "language": "French", "body": "I'm hungry" },
+                    "response": {
+                        "output": "J'ai faim.",
+                        "tokenUsage": { "cached": 19, "total": 19 },
+                        "cached": true,
+                        "cost": 0.0000046499999999999995
+                    },
+                    "success": true,
+                    "score": 1,
+                    "namedScores": {},
+                    "latencyMs": 2,
+                    "cost": 0.0000046499999999999995,
+                    "gradingResult": {
+                        "pass": true,
+                        "score": 1,
+                        "reason": "No assertions",
+                        "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                        "assertion": null
+                    }
+                },
+                {
+                    "provider": { "id": "openai:gpt-4o-mini", "label": "" },
+                    "prompt": {
+                        "raw": "Rephrase this in French: I'm hungry",
+                        "label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}"
+                    },
+                    "vars": { "language": "French", "body": "I'm hungry" },
+                    "response": {
+                        "output": "J'ai faim.",
+                        "tokenUsage": { "cached": 19, "total": 19 },
+                        "cached": true,
+                        "cost": 0.0000046499999999999995
+                    },
+                    "success": true,
+                    "score": 1,
+                    "namedScores": {},
+                    "latencyMs": 4,
+                    "cost": 0.0000046499999999999995,
+                    "gradingResult": {
+                        "pass": true,
+                        "score": 1,
+                        "reason": "No assertions",
+                        "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                        "assertion": null
+                    }
+                },
+                {
+                    "provider": { "id": "openai:gpt-4o-mini", "label": "" },
+                    "prompt": {
+                        "raw": "Rephrase this in French: Hello world",
+                        "label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}"
+                    },
+                    "vars": { "language": "French", "body": "Hello world" },
+                    "response": {
+                        "output": "Bonjour le monde.",
+                        "tokenUsage": { "cached": 19, "total": 19 },
+                        "cached": true,
+                        "cost": 0.0000046499999999999995
+                    },
+                    "success": true,
+                    "score": 1,
+                    "namedScores": {},
+                    "latencyMs": 6,
+                    "cost": 0.0000046499999999999995,
+                    "gradingResult": {
+                        "pass": true,
+                        "score": 1,
+                        "reason": "No assertions",
+                        "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                        "assertion": null
+                    }
+                },
+                {
+                    "provider": { "id": "openai:gpt-4o-mini", "label": "" },
+                    "prompt": {
+                        "raw": "Translate this to conversational French: Hello world",
+                        "label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}"
+                    },
+                    "vars": { "language": "French", "body": "Hello world" },
+                    "response": {
+                        "output": "Salut tout le monde !",
+                        "tokenUsage": { "cached": 20, "total": 20 },
+                        "cached": true,
+                        "cost": 0.0000052500000000000006
+                    },
+                    "success": true,
+                    "score": 1,
+                    "namedScores": {},
+                    "latencyMs": 6,
+                    "cost": 0.0000052500000000000006,
+                    "gradingResult": {
+                        "pass": true,
+                        "score": 1,
+                        "reason": "No assertions",
+                        "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                        "assertion": null
+                    }
+                },
+                {
+                    "provider": { "id": "openai:gpt-4o-mini", "label": "" },
+                    "prompt": {
+                        "raw": "Rephrase this in Pirate: Hello world",
+                        "label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}"
+                    },
+                    "vars": { "language": "Pirate", "body": "Hello world" },
+                    "response": {
+                        "output": "Ahoy, me hearties!",
+                        "tokenUsage": { "cached": 22, "total": 22 },
+                        "cached": true,
+                        "cost": 0.000006449999999999999
+                    },
+                    "success": true,
+                    "score": 1,
+                    "namedScores": {},
+                    "latencyMs": 3,
+                    "cost": 0.000006449999999999999,
+                    "gradingResult": {
+                        "pass": true,
+                        "score": 1,
+                        "reason": "No assertions",
+                        "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                        "assertion": null
+                    }
+                },
+                {
+                    "provider": { "id": "openai:gpt-4o-mini", "label": "" },
+                    "prompt": {
+                        "raw": "Translate this to conversational Pirate: Hello world",
+                        "label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}"
+                    },
+                    "vars": { "language": "Pirate", "body": "Hello world" },
+                    "response": {
+                        "output": "Ahoy, me hearty! Greetings to ye, world!",
+                        "tokenUsage": { "cached": 27, "total": 27 },
+                        "cached": true,
+                        "cost": 0.00000945
+                    },
+                    "success": true,
+                    "score": 1,
+                    "namedScores": {},
+                    "latencyMs": 3,
+                    "cost": 0.00000945,
+                    "gradingResult": {
+                        "pass": true,
+                        "score": 1,
+                        "reason": "No assertions",
+                        "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                        "assertion": null
+                    }
+                },
+                {
+                    "provider": { "id": "openai:gpt-4o-mini", "label": "" },
+                    "prompt": {
+                        "raw": "Rephrase this in Pirate: I'm hungry",
+                        "label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}"
+                    },
+                    "vars": { "language": "Pirate", "body": "I'm hungry" },
+                    "response": {
+                        "output": "Arrr, me belly be grumblin'!",
+                        "tokenUsage": { "cached": 27, "total": 27 },
+                        "cached": true,
+                        "cost": 0.00000945
+                    },
+                    "success": true,
+                    "score": 1,
+                    "namedScores": {},
+                    "latencyMs": 2,
+                    "cost": 0.00000945,
+                    "gradingResult": {
+                        "pass": true,
+                        "score": 1,
+                        "reason": "No assertions",
+                        "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                        "assertion": null
+                    }
+                },
+                {
+                    "provider": { "id": "openai:gpt-4o-mini", "label": "" },
+                    "prompt": {
+                        "raw": "Translate this to conversational Pirate: I'm hungry",
+                        "label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}"
+                    },
+                    "vars": { "language": "Pirate", "body": "I'm hungry" },
+                    "response": {
+                        "output": "Arrr, me belly be rumblin'! I be feelin' mighty hungry, matey!",
+                        "tokenUsage": { "cached": 37, "total": 37 },
+                        "cached": true,
+                        "cost": 0.00001545
+                    },
+                    "success": true,
+                    "score": 1,
+                    "namedScores": {},
+                    "latencyMs": 1,
+                    "cost": 0.00001545,
+                    "gradingResult": {
+                        "pass": true,
+                        "score": 1,
+                        "reason": "No assertions",
+                        "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                        "assertion": null
+                    }
+                }
+            ],
+            "stats": {
+                "successes": 8,
+                "failures": 0,
+                "tokenUsage": { "total": 190, "prompt": 0, "completion": 0, "cached": 190 }
+            },
+            "table": {
+                "head": {
+                    "prompts": [
+                        {
+                            "raw": "Rephrase this in {{language}}: {{body}}",
+                            "label": "examples/simple-csv/prompts.txt: Rephrase this in {{language}}: {{body}}",
+                            "id": "ad447337160016cbd0a5c7f2626d0c1fe2ade0d4d653e20bed9250632bc56e03",
+                            "provider": "openai:gpt-4o-mini",
+                            "metrics": {
+                                "score": 4,
+                                "testPassCount": 4,
+                                "testFailCount": 0,
+                                "assertPassCount": 0,
+                                "assertFailCount": 0,
+                                "totalLatencyMs": 15,
+                                "tokenUsage": { "total": 87, "prompt": 0, "completion": 0, "cached": 87 },
+                                "namedScores": {},
+                                "namedScoresCount": {},
+                                "cost": 0.000025199999999999996
+                            }
+                        },
+                        {
+                            "raw": "Translate this to conversational {{language}}: {{body}}",
+                            "label": "examples/simple-csv/prompts.txt: Translate this to conversational {{language}}: {{body}}",
+                            "id": "0ceccef5d40adcbf45bda7be485d2ee420e4d5943975932e7a38c5468d97c6ee",
+                            "provider": "openai:gpt-4o-mini",
+                            "metrics": {
+                                "score": 4,
+                                "testPassCount": 4,
+                                "testFailCount": 0,
+                                "assertPassCount": 0,
+                                "assertFailCount": 0,
+                                "totalLatencyMs": 12,
+                                "tokenUsage": { "total": 103, "prompt": 0, "completion": 0, "cached": 103 },
+                                "namedScores": {},
+                                "namedScoresCount": {},
+                                "cost": 0.0000348
+                            }
+                        }
+                    ],
+                    "vars": ["body", "language"]
+                },
+                "body": [
+                    {
+                        "description": "Row #1",
+                        "outputs": [
+                            {
+                                "pass": true,
+                                "score": 1,
+                                "namedScores": {},
+                                "text": "Bonjour le monde.",
+                                "prompt": "Rephrase this in French: Hello world",
+                                "provider": "openai:gpt-4o-mini",
+                                "latencyMs": 6,
+                                "tokenUsage": { "cached": 19, "total": 19 },
+                                "gradingResult": {
+                                    "pass": true,
+                                    "score": 1,
+                                    "reason": "No assertions",
+                                    "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                                    "assertion": null
+                                },
+                                "cost": 0.0000046499999999999995
+                            },
+                            {
+                                "pass": true,
+                                "score": 1,
+                                "namedScores": {},
+                                "text": "Salut tout le monde !",
+                                "prompt": "Translate this to conversational French: Hello world",
+                                "provider": "openai:gpt-4o-mini",
+                                "latencyMs": 6,
+                                "tokenUsage": { "cached": 20, "total": 20 },
+                                "gradingResult": {
+                                    "pass": true,
+                                    "score": 1,
+                                    "reason": "No assertions",
+                                    "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                                    "assertion": null
+                                },
+                                "cost": 0.0000052500000000000006
+                            }
+                        ],
+                        "test": {
+                            "vars": { "language": "French", "body": "Hello world" },
+                            "assert": [],
+                            "options": {},
+                            "description": "Row #1",
+                            "metadata": {}
+                        },
+                        "vars": ["Hello world", "French"]
+                    },
+                    {
+                        "description": "Row #2",
+                        "outputs": [
+                            {
+                                "pass": true,
+                                "score": 1,
+                                "namedScores": {},
+                                "text": "J'ai faim.",
+                                "prompt": "Rephrase this in French: I'm hungry",
+                                "provider": "openai:gpt-4o-mini",
+                                "latencyMs": 4,
+                                "tokenUsage": { "cached": 19, "total": 19 },
+                                "gradingResult": {
+                                    "pass": true,
+                                    "score": 1,
+                                    "reason": "No assertions",
+                                    "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                                    "assertion": null
+                                },
+                                "cost": 0.0000046499999999999995
+                            },
+                            {
+                                "pass": true,
+                                "score": 1,
+                                "namedScores": {},
+                                "text": "J'ai faim.",
+                                "prompt": "Translate this to conversational French: I'm hungry",
+                                "provider": "openai:gpt-4o-mini",
+                                "latencyMs": 2,
+                                "tokenUsage": { "cached": 19, "total": 19 },
+                                "gradingResult": {
+                                    "pass": true,
+                                    "score": 1,
+                                    "reason": "No assertions",
+                                    "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                                    "assertion": null
+                                },
+                                "cost": 0.0000046499999999999995
+                            }
+                        ],
+                        "test": {
+                            "vars": { "language": "French", "body": "I'm hungry" },
+                            "assert": [],
+                            "options": {},
+                            "description": "Row #2",
+                            "metadata": {}
+                        },
+                        "vars": ["I'm hungry", "French"]
+                    },
+                    {
+                        "description": "Row #3",
+                        "outputs": [
+                            {
+                                "pass": true,
+                                "score": 1,
+                                "namedScores": {},
+                                "text": "Ahoy, me hearties!",
+                                "prompt": "Rephrase this in Pirate: Hello world",
+                                "provider": "openai:gpt-4o-mini",
+                                "latencyMs": 3,
+                                "tokenUsage": { "cached": 22, "total": 22 },
+                                "gradingResult": {
+                                    "pass": true,
+                                    "score": 1,
+                                    "reason": "No assertions",
+                                    "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                                    "assertion": null
+                                },
+                                "cost": 0.000006449999999999999
+                            },
+                            {
+                                "pass": true,
+                                "score": 1,
+                                "namedScores": {},
+                                "text": "Ahoy, me hearty! Greetings to ye, world!",
+                                "prompt": "Translate this to conversational Pirate: Hello world",
+                                "provider": "openai:gpt-4o-mini",
+                                "latencyMs": 3,
+                                "tokenUsage": { "cached": 27, "total": 27 },
+                                "gradingResult": {
+                                    "pass": true,
+                                    "score": 1,
+                                    "reason": "No assertions",
+                                    "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                                    "assertion": null
+                                },
+                                "cost": 0.00000945
+                            }
+                        ],
+                        "test": {
+                            "vars": { "language": "Pirate", "body": "Hello world" },
+                            "assert": [],
+                            "options": {},
+                            "description": "Row #3",
+                            "metadata": {}
+                        },
+                        "vars": ["Hello world", "Pirate"]
+                    },
+                    {
+                        "description": "Row #4",
+                        "outputs": [
+                            {
+                                "pass": true,
+                                "score": 1,
+                                "namedScores": {},
+                                "text": "Arrr, me belly be grumblin'!",
+                                "prompt": "Rephrase this in Pirate: I'm hungry",
+                                "provider": "openai:gpt-4o-mini",
+                                "latencyMs": 2,
+                                "tokenUsage": { "cached": 27, "total": 27 },
+                                "gradingResult": {
+                                    "pass": true,
+                                    "score": 1,
+                                    "reason": "No assertions",
+                                    "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                                    "assertion": null
+                                },
+                                "cost": 0.00000945
+                            },
+                            {
+                                "pass": true,
+                                "score": 1,
+                                "namedScores": {},
+                                "text": "Arrr, me belly be rumblin'! I be feelin' mighty hungry, matey!",
+                                "prompt": "Translate this to conversational Pirate: I'm hungry",
+                                "provider": "openai:gpt-4o-mini",
+                                "latencyMs": 1,
+                                "tokenUsage": { "cached": 37, "total": 37 },
+                                "gradingResult": {
+                                    "pass": true,
+                                    "score": 1,
+                                    "reason": "No assertions",
+                                    "tokensUsed": { "total": 0, "prompt": 0, "completion": 0, "cached": 0 },
+                                    "assertion": null
+                                },
+                                "cost": 0.00001545
+                            }
+                        ],
+                        "test": {
+                            "vars": { "language": "Pirate", "body": "I'm hungry" },
+                            "assert": [],
+                            "options": {},
+                            "description": "Row #4",
+                            "metadata": {}
+                        },
+                        "vars": ["I'm hungry", "Pirate"]
+                    }
+                ]
+            }
+        },
+        "config": {
+            "tags": {},
+            "description": "A translator built with LLM",
+            "prompts": ["file:///Users/steve/src/promptfoo/examples/simple-csv/prompts.txt"],
+            "providers": ["openai:gpt-4o-mini"],
+            "tests": [
+                {
+                    "vars": { "language": "French", "body": "Hello world" },
+                    "assert": [],
+                    "options": {},
+                    "description": "Row #1"
+                },
+                {
+                    "vars": { "language": "French", "body": "I'm hungry" },
+                    "assert": [],
+                    "options": {},
+                    "description": "Row #2"
+                },
+                {
+                    "vars": { "language": "Pirate", "body": "Hello world" },
+                    "assert": [],
+                    "options": {},
+                    "description": "Row #3"
+                },
+                {
+                    "vars": { "language": "Pirate", "body": "I'm hungry" },
+                    "assert": [],
+                    "options": {},
+                    "description": "Row #4"
+                }
+            ],
+            "scenarios": [],
+            "env": {},
+            "sharing": true,
+            "defaultTest": { "vars": {}, "assert": [], "options": {}, "metadata": {} },
+            "outputPath": [],
+            "extensions": [],
+            "metadata": {},
+            "redteam": { "plugins": [], "strategies": [] }
+        }
+    }
+}