promptfoo 0.20.0 → 0.21.0

package/src/web/nextui/src/app/setup/TestCaseDialog.tsx

@@ -1,108 +0,0 @@
-import React, { useState } from 'react';
-import {
-  Button,
-  TextField,
-  Box,
-  Dialog,
-  DialogTitle,
-  DialogContent,
-  DialogActions,
-} from '@mui/material';
-import VarsForm from './VarsForm';
-import AssertsForm from './AssertsForm';
-import type { TestCase } from '../../../../../types';
-
-interface TestCaseFormProps {
-  open: boolean;
-  onAdd: (testCase: TestCase, shouldClose: boolean) => void;
-  varsList: string[];
-  initialValues?: TestCase;
-  onCancel: () => void;
-}
-
-const TestCaseForm: React.FC<TestCaseFormProps> = ({
-  open,
-  onAdd,
-  varsList,
-  initialValues,
-  onCancel,
-}) => {
-  const [description, setDescription] = useState(initialValues?.description || '');
-  const [vars, setVars] = useState(initialValues?.vars || {});
-  const [asserts, setAsserts] = useState(initialValues?.assert || []);
-  const [assertsFormKey, setAssertsFormKey] = useState(0);
-
-  React.useEffect(() => {
-    if (initialValues) {
-      setDescription(initialValues.description || '');
-      setVars(initialValues.vars || {});
-      setAsserts(initialValues.assert || []);
-    } else {
-      setDescription('');
-      setVars({});
-      setAsserts([]);
-    }
-  }, [initialValues]);
-
-  const handleAdd = (close: boolean) => {
-    onAdd(
-      {
-        description,
-        vars,
-        assert: asserts,
-      },
-      close,
-    );
-    if (close) {
-      onCancel();
-    }
-    setDescription('');
-    setVars({});
-    setAsserts([]);
-    setAssertsFormKey((prevKey) => prevKey + 1);
-  };
-
-  return (
-    <Dialog open={open} onClose={onCancel} fullWidth maxWidth="md">
-      <DialogTitle>{initialValues ? 'Edit Test Case' : 'Add Test Case'}</DialogTitle>
-      <DialogContent>
-        <Box>
-          {/*
-          <TextField
-            label="Description"
-            value={description}
-            onChange={(e) => setDescription(e.target.value)}
-            fullWidth
-            margin="normal"
-          />
-          */}
-          <VarsForm
-            onAdd={(vars) => setVars(vars)}
-            varsList={varsList}
-            initialValues={initialValues?.vars as Record<string, string>}
-          />
-          <AssertsForm
-            key={assertsFormKey}
-            onAdd={(asserts) => setAsserts(asserts)}
-            initialValues={initialValues?.assert || []}
-          />
-        </Box>
-      </DialogContent>
-      <DialogActions>
-        <Button onClick={handleAdd.bind(this, true)} color="primary" variant="contained">
-          {initialValues ? 'Update Test Case' : 'Add Test Case'}
-        </Button>
-        {!initialValues && (
-          <Button onClick={handleAdd.bind(this, false)} color="primary" variant="contained">
-            Add Another
-          </Button>
-        )}
-        <Button onClick={onCancel} color="secondary">
-          Cancel
-        </Button>
-      </DialogActions>
-    </Dialog>
-  );
-};
-
-export default TestCaseForm;

package/src/web/nextui/src/app/setup/TestCasesSection.tsx

@@ -1,154 +0,0 @@
-import React from 'react';
-import Button from '@mui/material/Button';
-import Copy from '@mui/icons-material/ContentCopy';
-import Delete from '@mui/icons-material/Delete';
-import Edit from '@mui/icons-material/Edit';
-import IconButton from '@mui/material/IconButton';
-import Stack from '@mui/material/Stack';
-import Table from '@mui/material/Table';
-import TableBody from '@mui/material/TableBody';
-import TableCell from '@mui/material/TableCell';
-import TableContainer from '@mui/material/TableContainer';
-import TableHead from '@mui/material/TableHead';
-import TableRow from '@mui/material/TableRow';
-import Typography from '@mui/material/Typography';
-
-import TestCaseDialog from './TestCaseDialog';
-import { useStore } from '../../util/store';
-
-import type { TestCase } from '../../../../../types';
-
-interface TestCasesSectionProps {
-  varsList: string[];
-}
-
-const TestCasesSection: React.FC<TestCasesSectionProps> = ({ varsList }) => {
-  const { testCases, setTestCases } = useStore();
-  const [editingTestCaseIndex, setEditingTestCaseIndex] = React.useState<number | null>(null);
-  const [testCaseDialogOpen, setTestCaseDialogOpen] = React.useState(false);
-
-  const handleAddTestCase = (testCase: TestCase, shouldClose: boolean) => {
-    if (editingTestCaseIndex === null) {
-      setTestCases([...testCases, testCase]);
-    } else {
-      const updatedTestCases = testCases.map((tc, index) =>
-        index === editingTestCaseIndex ? testCase : tc,
-      );
-      setTestCases(updatedTestCases);
-      setEditingTestCaseIndex(null);
-    }
-
-    if (shouldClose) {
-      setTestCaseDialogOpen(false);
-    }
-  };
-
-  const handleRemoveTestCase = (event: React.MouseEvent, index: number) => {
-    event.stopPropagation();
-
-    if (confirm('Are you sure you want to delete this test case?')) {
-      setTestCases(testCases.filter((_, i) => i !== index));
-    }
-  };
-
-  const handleDuplicateTestCase = (event: React.MouseEvent, index: number) => {
-    event.stopPropagation();
-    const duplicatedTestCase = JSON.parse(JSON.stringify(testCases[index]));
-    setTestCases([...testCases, duplicatedTestCase]);
-  };
-
-  return (
-    <>
-      <Stack direction="row" spacing={2} justifyContent="space-between">
-        <Typography variant="h5">Test Cases</Typography>
-        <Button color="primary" onClick={() => setTestCaseDialogOpen(true)} variant="contained">
-          Add Test Case
-        </Button>
-      </Stack>
-      <TableContainer>
-        <Table>
-          <TableHead>
-            <TableRow>
-              <TableCell>Description</TableCell>
-              <TableCell>Assertions</TableCell>
-              <TableCell>Variables</TableCell>
-              <TableCell align="right"></TableCell>
-            </TableRow>
-          </TableHead>
-          <TableBody>
-            {testCases.length === 0 ? (
-              <TableRow>
-                <TableCell colSpan={4} align="center">
-                  No test cases added yet.
-                </TableCell>
-              </TableRow>
-            ) : (
-              testCases.map((testCase, index) => (
-                <TableRow
-                  key={index}
-                  sx={{
-                    '&:hover': {
-                      backgroundColor: 'rgba(0, 0, 0, 0.04)',
-                      cursor: 'pointer',
-                    },
-                  }}
-                  onClick={() => {
-                    setEditingTestCaseIndex(index);
-                    setTestCaseDialogOpen(true);
-                  }}
-                >
-                  <TableCell>
-                    <Typography variant="body2">
-                      {testCase.description || `Test Case #${index + 1}`}
-                    </Typography>
-                  </TableCell>
-                  <TableCell>{testCase.assert?.length || 0} assertions</TableCell>
-                  <TableCell>
-                    {Object.entries(testCase.vars || {})
-                      .map(([k, v]) => k + '=' + v)
-                      .join(', ')}
-                  </TableCell>
-                  <TableCell align="right" sx={{ minWidth: 150 }}>
-                    <IconButton
-                      onClick={() => {
-                        setEditingTestCaseIndex(index);
-                        setTestCaseDialogOpen(true);
-                      }}
-                      size="small"
-                    >
-                      <Edit />
-                    </IconButton>
-                    <IconButton
-                      onClick={(event) => handleDuplicateTestCase(event, index)}
-                      size="small"
-                    >
-                      <Copy />
-                    </IconButton>
-                    <IconButton
-                      onClick={(event) => handleRemoveTestCase(event, index)}
-                      size="small"
-                    >
-                      <Delete />
-                    </IconButton>
-                  </TableCell>
-                </TableRow>
-              ))
-            )}
-          </TableBody>
-        </Table>
-      </TableContainer>
-      <TestCaseDialog
-        open={testCaseDialogOpen}
-        onAdd={handleAddTestCase}
-        varsList={varsList}
-        initialValues={editingTestCaseIndex !== null ? testCases[editingTestCaseIndex] : undefined}
-        onCancel={() => {
-          setEditingTestCaseIndex(null);
-          setTestCaseDialogOpen(false);
-        }}
-      />
-    </>
-  );
-};
-
-export default TestCasesSection;

package/src/web/nextui/src/app/setup/VarsForm.tsx

@@ -1,57 +0,0 @@
-import React, { useEffect } from 'react';
-import { Box, TextField, Typography, Stack } from '@mui/material';
-
-interface VarsFormProps {
-  onAdd: (vars: Record<string, string>) => void;
-  varsList: string[];
-  initialValues?: Record<string, string>;
-}
-
-const VarsForm: React.FC<VarsFormProps> = ({ onAdd, varsList, initialValues }) => {
-  const [vars, setVars] = React.useState<Record<string, string>>(initialValues || {});
-
-  useEffect(() => {
-    const newVars: Record<string, string> = {};
-    varsList.forEach((v) => {
-      newVars[v] = initialValues?.[v] || '';
-    });
-    setVars(newVars);
-  }, [varsList, initialValues]);
-
-  return (
-    <Box my={2}>
-      <Typography variant="h6" mb={2}>
-        Vars
-      </Typography>
-      {varsList.length > 0 ? (
-        <Stack direction="row" spacing={2} alignItems="center">
-          {Object.keys(vars).map((varName, index) => (
-            <Stack key={index} direction="row" spacing={2} alignItems="center">
-              <TextField
-                placeholder={varName}
-                label={varName}
-                value={vars[varName]}
-                fullWidth
-                onChange={(e) => {
-                  const newValue = e.target.value;
-                  const newVars = {
-                    ...vars,
-                    [varName]: newValue,
-                  };
-                  setVars(newVars);
-                  onAdd(newVars);
-                }}
-              />
-            </Stack>
-          ))}
-        </Stack>
-      ) : (
-        <Typography variant="subtitle1" gutterBottom>
-          Add variables to your prompt using the {'{{varname}}'} syntax.
-        </Typography>
-      )}
-    </Box>
-  );
-};
-
-export default VarsForm;

package/src/web/nextui/src/app/setup/page.css

@@ -1,3 +0,0 @@
-.yaml-config {
-  font-size: 10px;
-}

package/src/web/nextui/src/app/setup/page.tsx

@@ -1,160 +0,0 @@
-'use client';
-
-import React, { useState, useEffect } from 'react';
-import Link from 'next/link';
-import yaml from 'js-yaml';
-import { Light as SyntaxHighlighter } from 'react-syntax-highlighter';
-import { docco } from 'react-syntax-highlighter/dist/cjs/styles/hljs';
-import Button from '@mui/material/Button';
-import Container from '@mui/material/Container';
-import Typography from '@mui/material/Typography';
-import Box from '@mui/material/Box';
-import Stack from '@mui/material/Stack';
-import Dialog from '@mui/material/Dialog';
-import DialogActions from '@mui/material/DialogActions';
-import DialogContent from '@mui/material/DialogContent';
-import DialogContentText from '@mui/material/DialogContentText';
-import DialogTitle from '@mui/material/DialogTitle';
-
-import RunTestSuiteButton from './RunTestSuiteButton';
-import PromptsSection from './PromptsSection';
-import TestCasesSection from './TestCasesSection';
-import ProviderSelector from './ProviderSelector';
-import { useStore } from '../../util/store';
-
-import './page.css';
-
-const EvaluateTestSuiteCreator: React.FC = () => {
-  const [yamlString, setYamlString] = useState('');
-  const [resetDialogOpen, setResetDialogOpen] = useState(false);
-
-  const {
-    description,
-    setDescription,
-    providers,
-    setProviders,
-    prompts,
-    setPrompts,
-    testCases,
-    setTestCases,
-  } = useStore();
-
-  useEffect(() => {
-    useStore.persist.rehydrate();
-  }, []);
-
-  useEffect(() => {
-    const testSuite = {
-      description,
-      providers,
-      prompts,
-      tests: testCases,
-    };
-    setYamlString(yaml.dump(testSuite));
-  }, [description, providers, prompts, testCases]);
-
-  if (process.env.NEXT_PUBLIC_NO_BROWSING) {
-    return null;
-  }
-
-  const extractVarsFromPrompts = (prompts: string[]): string[] => {
-    const varRegex = /{{(\w+)}}/g;
-    const varsSet = new Set<string>();
-
-    prompts.forEach((prompt) => {
-      let match;
-      while ((match = varRegex.exec(prompt)) !== null) {
-        varsSet.add(match[1]);
-      }
-    });
-
-    return Array.from(varsSet);
-  };
-
-  const varsList = extractVarsFromPrompts(prompts);
-
-  const handleReset = () => {
-    setDescription('');
-    setProviders([]);
-    setPrompts([]);
-    setTestCases([]);
-    setYamlString('');
-    setResetDialogOpen(false);
-  };
-
-  return (
-    <Container maxWidth="lg" sx={{ marginTop: '2rem' }}>
-      <Stack direction="row" spacing={2} justifyContent="space-between">
-        <Typography variant="h4">Set up an evaluation</Typography>
-        <Stack direction="row" spacing={2}>
-          <RunTestSuiteButton />
-          <Button variant="outlined" color="primary" onClick={() => setResetDialogOpen(true)}>
-            Reset
-          </Button>
-        </Stack>
-      </Stack>
-      <Box mt={4} />
-      {/*
-      <Box mt={4}>
-        <TextField
-          label="Description"
-          value={description}
-          onChange={(e) => {
-            setDescription(e.target.value);
-          }}
-          fullWidth
-          margin="normal"
-        />
-      </Box>
-      */}
-      <Box mt={2}>
-        <Stack direction="column" spacing={2} justifyContent="space-between">
-          <Typography variant="h5">Providers</Typography>
-          <ProviderSelector providers={providers} onChange={setProviders} />
-        </Stack>
-      </Box>
-      <Box mt={4} />
-      <PromptsSection />
-      <Box mt={6} />
-      <TestCasesSection varsList={varsList} />
-      <Box mt={8}>
-        {yamlString && (
-          <Box mt={4}>
-            <Typography variant="h5" gutterBottom>
-              YAML config
-            </Typography>
-            <Typography variant="body1" gutterBottom>
-              This is the evaluation config that is run by promptfoo. See{' '}
-              <Link href="https://promptfoo.dev/docs/configuration/guide">configuration docs</Link>{' '}
-              to learn more.
-            </Typography>
-            <SyntaxHighlighter className="yaml-config" language="yaml" style={docco}>
-              {yamlString}
-            </SyntaxHighlighter>
-          </Box>
-        )}
-      </Box>
-      <Dialog
-        open={resetDialogOpen}
-        onClose={() => setResetDialogOpen(false)}
-        aria-labelledby="alert-dialog-title"
-        aria-describedby="alert-dialog-description"
-      >
-        <DialogTitle id="alert-dialog-title">{'Confirm Reset'}</DialogTitle>
-        <DialogContent>
-          <DialogContentText id="alert-dialog-description">
-            Are you sure you want to reset all the fields? This action cannot be undone.
-          </DialogContentText>
-        </DialogContent>
-        <DialogActions>
-          <Button onClick={() => setResetDialogOpen(false)}>Cancel</Button>
-          <Button onClick={handleReset} autoFocus>
-            Reset
-          </Button>
-        </DialogActions>
-      </Dialog>
-    </Container>
-  );
-};
-
-export default EvaluateTestSuiteCreator;

package/src/web/nextui/src/util/api.ts

@@ -1 +0,0 @@
-export const API_BASE_URL = `http://localhost:15500`;

package/src/web/nextui/src/util/store.ts

@@ -1,53 +0,0 @@
-import { create } from 'zustand';
-import { persist } from 'zustand/middleware';
-
-import type { Assertion, ProviderOptions, TestCase } from '../../../../types';
-
-export interface State {
-  asserts: Assertion[];
-  testCases: TestCase[];
-  description: string;
-  providers: ProviderOptions[];
-  prompts: string[];
-  setAsserts: (asserts: Assertion[]) => void;
-  setTestCases: (testCases: TestCase[]) => void;
-  setDescription: (description: string) => void;
-  setProviders: (providers: ProviderOptions[]) => void;
-  setPrompts: (prompts: string[]) => void;
-}
-
-export const useStore = create<State>()(
-  persist(
-    (set) => ({
-      asserts: [],
-      testCases: [],
-      description: '',
-      providers: [],
-      prompts: [],
-      setAsserts: (asserts) => set({ asserts }),
-      setTestCases: (testCases) => set({ testCases }),
-      setDescription: (description) => set({ description }),
-      setProviders: (providers) => set({ providers }),
-      setPrompts: (prompts) => set({ prompts }),
-    }),
-    {
-      name: 'promptfoo',
-      skipHydration: true,
-    },
-  ),
-);
-
-/*
-export const useStore = create<State>((set) => ({
-  asserts: [],
-  testCases: [],
-  description: '',
-  providers: [],
-  prompts: [],
-  setAsserts: (asserts) => set({ asserts }),
-  setTestCases: (testCases) => set({ testCases }),
-  setDescription: (description) => set({ description }),
-  setProviders: (providers) => set({ providers }),
-  setPrompts: (prompts) => set({ prompts }),
-}));
-*/

package/src/web/nextui/tsconfig.json

@@ -1,28 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "es5",
-    "lib": ["dom", "dom.iterable", "esnext"],
-    "allowJs": true,
-    "skipLibCheck": true,
-    "strict": true,
-    "forceConsistentCasingInFileNames": true,
-    "noEmit": true,
-    "esModuleInterop": true,
-    "module": "esnext",
-    "moduleResolution": "bundler",
-    "resolveJsonModule": true,
-    "isolatedModules": true,
-    "jsx": "preserve",
-    "incremental": true,
-    "plugins": [
-      {
-        "name": "next"
-      }
-    ],
-    "paths": {
-      "@/*": ["./src/*"]
-    }
-  },
-  "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
-  "exclude": ["node_modules"]
-}