promptfoo 0.18.4 → 0.19.0

package/src/web/nextui/src/app/setup/page.tsx

@@ -0,0 +1,160 @@
+'use client';
+
+import React, { useState, useEffect } from 'react';
+import Link from 'next/link';
+import yaml from 'js-yaml';
+import { Light as SyntaxHighlighter } from 'react-syntax-highlighter';
+import { docco } from 'react-syntax-highlighter/dist/cjs/styles/hljs';
+import Button from '@mui/material/Button';
+import Container from '@mui/material/Container';
+import Typography from '@mui/material/Typography';
+import Box from '@mui/material/Box';
+import Stack from '@mui/material/Stack';
+import Dialog from '@mui/material/Dialog';
+import DialogActions from '@mui/material/DialogActions';
+import DialogContent from '@mui/material/DialogContent';
+import DialogContentText from '@mui/material/DialogContentText';
+import DialogTitle from '@mui/material/DialogTitle';
+
+import RunTestSuiteButton from './RunTestSuiteButton';
+import PromptsSection from './PromptsSection';
+import TestCasesSection from './TestCasesSection';
+import ProviderSelector from './ProviderSelector';
+import { useStore } from '../../util/store';
+
+import './page.css';
+
+const EvaluateTestSuiteCreator: React.FC = () => {
+  const [yamlString, setYamlString] = useState('');
+  const [resetDialogOpen, setResetDialogOpen] = useState(false);
+
+  const {
+    description,
+    setDescription,
+    providers,
+    setProviders,
+    prompts,
+    setPrompts,
+    testCases,
+    setTestCases,
+  } = useStore();
+
+  useEffect(() => {
+    useStore.persist.rehydrate();
+  }, []);
+
+  useEffect(() => {
+    const testSuite = {
+      description,
+      providers,
+      prompts,
+      tests: testCases,
+    };
+    setYamlString(yaml.dump(testSuite));
+  }, [description, providers, prompts, testCases]);
+
+  if (process.env.NEXT_PUBLIC_NO_BROWSING) {
+    return null;
+  }
+
+  const extractVarsFromPrompts = (prompts: string[]): string[] => {
+    const varRegex = /{{(\w+)}}/g;
+    const varsSet = new Set<string>();
+
+    prompts.forEach((prompt) => {
+      let match;
+      while ((match = varRegex.exec(prompt)) !== null) {
+        varsSet.add(match[1]);
+      }
+    });
+
+    return Array.from(varsSet);
+  };
+
+  const varsList = extractVarsFromPrompts(prompts);
+
+  const handleReset = () => {
+    setDescription('');
+    setProviders([]);
+    setPrompts([]);
+    setTestCases([]);
+    setYamlString('');
+    setResetDialogOpen(false);
+  };
+
+  return (
+    <Container maxWidth="lg" sx={{ marginTop: '2rem' }}>
+      <Stack direction="row" spacing={2} justifyContent="space-between">
+        <Typography variant="h4">Set up an evaluation</Typography>
+        <Stack direction="row" spacing={2}>
+          <RunTestSuiteButton />
+          <Button variant="outlined" color="primary" onClick={() => setResetDialogOpen(true)}>
+            Reset
+          </Button>
+        </Stack>
+      </Stack>
+      <Box mt={4} />
+      {/*
+      <Box mt={4}>
+        <TextField
+          label="Description"
+          value={description}
+          onChange={(e) => {
+            setDescription(e.target.value);
+          }}
+          fullWidth
+          margin="normal"
+        />
+      </Box>
+      */}
+      <Box mt={2}>
+        <Stack direction="column" spacing={2} justifyContent="space-between">
+          <Typography variant="h5">Providers</Typography>
+          <ProviderSelector providers={providers} onChange={setProviders} />
+        </Stack>
+      </Box>
+      <Box mt={4} />
+      <PromptsSection />
+      <Box mt={6} />
+      <TestCasesSection varsList={varsList} />
+      <Box mt={8}>
+        {yamlString && (
+          <Box mt={4}>
+            <Typography variant="h5" gutterBottom>
+              YAML config
+            </Typography>
+            <Typography variant="body1" gutterBottom>
+              This is the evaluation config that is run by promptfoo. See{' '}
+              <Link href="https://promptfoo.dev/docs/configuration/guide">configuration docs</Link>{' '}
+              to learn more.
+            </Typography>
+            <SyntaxHighlighter className="yaml-config" language="yaml" style={docco}>
+              {yamlString}
+            </SyntaxHighlighter>
+          </Box>
+        )}
+      </Box>
+      <Dialog
+        open={resetDialogOpen}
+        onClose={() => setResetDialogOpen(false)}
+        aria-labelledby="alert-dialog-title"
+        aria-describedby="alert-dialog-description"
+      >
+        <DialogTitle id="alert-dialog-title">{'Confirm Reset'}</DialogTitle>
+        <DialogContent>
+          <DialogContentText id="alert-dialog-description">
+            Are you sure you want to reset all the fields? This action cannot be undone.
+          </DialogContentText>
+        </DialogContent>
+        <DialogActions>
+          <Button onClick={() => setResetDialogOpen(false)}>Cancel</Button>
+          <Button onClick={handleReset} autoFocus>
+            Reset
+          </Button>
+        </DialogActions>
+      </Dialog>
+    </Container>
+  );
+};
+
+export default EvaluateTestSuiteCreator;

package/src/web/nextui/src/util/api.ts

@@ -0,0 +1 @@
+export const API_BASE_URL = `http://localhost:15500`;

package/src/web/nextui/src/util/store.ts

@@ -0,0 +1,53 @@
+import { create } from 'zustand';
+import { persist } from 'zustand/middleware';
+
+import type { Assertion, ProviderConfig, TestCase } from '../../../../types';
+
+export interface State {
+  asserts: Assertion[];
+  testCases: TestCase[];
+  description: string;
+  providers: ProviderConfig[];
+  prompts: string[];
+  setAsserts: (asserts: Assertion[]) => void;
+  setTestCases: (testCases: TestCase[]) => void;
+  setDescription: (description: string) => void;
+  setProviders: (providers: ProviderConfig[]) => void;
+  setPrompts: (prompts: string[]) => void;
+}
+
+export const useStore = create<State>()(
+  persist(
+    (set) => ({
+      asserts: [],
+      testCases: [],
+      description: '',
+      providers: [],
+      prompts: [],
+      setAsserts: (asserts) => set({ asserts }),
+      setTestCases: (testCases) => set({ testCases }),
+      setDescription: (description) => set({ description }),
+      setProviders: (providers) => set({ providers }),
+      setPrompts: (prompts) => set({ prompts }),
+    }),
+    {
+      name: 'promptfoo',
+      skipHydration: true,
+    },
+  ),
+);
+
+/*
+export const useStore = create<State>((set) => ({
+  asserts: [],
+  testCases: [],
+  description: '',
+  providers: [],
+  prompts: [],
+  setAsserts: (asserts) => set({ asserts }),
+  setTestCases: (testCases) => set({ testCases }),
+  setDescription: (description) => set({ description }),
+  setProviders: (providers) => set({ providers }),
+  setPrompts: (prompts) => set({ prompts }),
+}));
+*/

package/src/web/nextui/tsconfig.json

@@ -0,0 +1,28 @@
+{
+  "compilerOptions": {
+    "target": "es5",
+    "lib": ["dom", "dom.iterable", "esnext"],
+    "allowJs": true,
+    "skipLibCheck": true,
+    "strict": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true,
+    "esModuleInterop": true,
+    "module": "esnext",
+    "moduleResolution": "bundler",
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "jsx": "preserve",
+    "incremental": true,
+    "plugins": [
+      {
+        "name": "next"
+      }
+    ],
+    "paths": {
+      "@/*": ["./src/*"]
+    }
+  },
+  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
+  "exclude": ["node_modules"]
+}

package/src/web/server.ts

@@ -2,23 +2,37 @@ import fs, { Stats } from 'fs';
 import path from 'node:path';
 import readline from 'node:readline';
 import http from 'node:http';
+import invariant from 'tiny-invariant';
+import { v4 as uuidv4 } from 'uuid';
 
 import debounce from 'debounce';
 import express from 'express';
 import cors from 'cors';
+import compression from 'compression';
 import opener from 'opener';
 import { Server as SocketIOServer } from 'socket.io';
+import promptfoo, { EvaluateSummary } from '../index';
 
 import logger from '../logger';
 import { getDirectory } from '../esm';
 import { getLatestResultsPath, listPreviousResults, readResult } from '../util';
 
-export function init(port = 15500) {
+interface Job {
+  status: 'in-progress' | 'completed';
+  progress: number;
+  total: number;
+  result: EvaluateSummary | null;
+}
+
+const evalJobs = new Map<string, Job>();
+
+export function startServer(port = 15500) {
   const app = express();
 
-  const staticDir = path.join(getDirectory(), 'web', 'client');
+  const staticDir = path.join(getDirectory(), 'web', 'nextui');
 
   app.use(cors());
+  app.use(compression());
   app.use(express.json());
   app.use(express.static(staticDir));
 
@@ -55,9 +69,49 @@ export function init(port = 15500) {
 
   app.get('/results', (req, res) => {
     const previousResults = listPreviousResults();
+    previousResults.reverse();
     res.json({ data: previousResults });
   });
 
+  app.post('/api/eval', (req, res) => {
+    const testSuite = req.body;
+    const id = uuidv4();
+    evalJobs.set(id, { status: 'in-progress', progress: 0, total: 0, result: null });
+
+    promptfoo
+      .evaluate(Object.assign({}, testSuite, { writeLatestResults: true }), {
+        progressCallback: (progress, total) => {
+          const job = evalJobs.get(id);
+          invariant(job, 'Job not found');
+          job.progress = progress;
+          job.total = total;
+          console.log(`Progress: ${progress}/${total}`);
+        },
+      })
+      .then((result) => {
+        const job = evalJobs.get(id);
+        invariant(job, 'Job not found');
+        job.status = 'completed';
+        job.result = result;
+      });
+
+    res.json({ id });
+  });
+
+  app.get('/api/eval/:id', (req, res) => {
+    const id = req.params.id;
+    const job = evalJobs.get(id);
+    if (!job) {
+      res.status(404).json({ error: 'Job not found' });
+      return;
+    }
+    if (job.status === 'completed') {
+      res.json({ status: 'completed', result: job.result });
+    } else {
+      res.json({ status: 'in-progress', progress: job.progress, total: job.total });
+    }
+  });
+
   app.get('/results/:filename', (req, res) => {
     const filename = req.params.filename;
     const safeFilename = path.basename(filename);