promptfoo 0.18.4 → 0.19.0

package/src/web/nextui/src/app/setup/ProviderSelector.tsx

@@ -0,0 +1,149 @@
+import React from 'react';
+import { Autocomplete, Box, Chip, TextField } from '@mui/material';
+import { ProviderConfig } from '../../../../../types';
+import ProviderConfigDialog from './ProviderConfigDialog';
+
+const defaultProviders: ProviderConfig[] = [
+  {
+    id: 'replicate:replicate/llama70b-v2-chat:e951f18578850b652510200860fc4ea62b3b16fac280f83ff32282f87bbd2e48',
+    config: { temperature: 0.5 },
+  },
+]
+  .concat(
+    [
+      'anthropic:claude-1',
+      'anthropic:claude-1-100k',
+      'anthropic:claude-instant-1',
+      'anthropic:claude-instant-1-100k',
+    ].map((id) => ({ id, config: { temperature: 0.5 } })),
+  )
+  .concat(
+    [
+      'openai:gpt-3.5-turbo',
+      'openai:gpt-3.5-turbo-0301',
+      'openai:gpt-3.5-turbo-0613',
+      'openai:gpt-3.5-turbo-16k',
+      'openai:gpt-3.5-turbo-16k-0613',
+      'openai:gpt-4',
+      'openai:gpt-4-0314',
+      'openai:gpt-4-0613',
+      'openai:gpt-4-32k',
+      'openai:gpt-4-32k-0314',
+    ].map((id) => ({ id, config: { temperature: 0.5, max_tokens: 1024 } })),
+  )
+  .concat(
+    [
+      'azureopenai:gpt-3.5-turbo',
+      'azureopenai:gpt-3.5-turbo-0301',
+      'azureopenai:gpt-3.5-turbo-0613',
+      'azureopenai:gpt-3.5-turbo-16k',
+      'azureopenai:gpt-3.5-turbo-16k-0613',
+      'azureopenai:gpt-4',
+      'azureopenai:gpt-4-0314',
+      'azureopenai:gpt-4-0613',
+      'azureopenai:gpt-4-32k',
+      'azureopenai:gpt-4-32k-0314',
+    ].map((id) => ({ id, config: { temperature: 0.5, max_tokens: 1024 } })),
+  )
+  .sort((a, b) => a.id.localeCompare(b.id));
+
+interface ProviderSelectorProps {
+  providers: ProviderConfig[];
+  onChange: (providers: ProviderConfig[]) => void;
+}
+
+const ProviderSelector: React.FC<ProviderSelectorProps> = ({ providers, onChange }) => {
+  const [selectedProvider, setSelectedProvider] = React.useState<ProviderConfig | null>(null);
+
+  const getProviderLabel = (provider: string | ProviderConfig) => {
+    if (typeof provider === 'string') {
+      return provider;
+    }
+    return provider.id || 'Unknown provider';
+  };
+
+  const getProviderKey = (provider: string | ProviderConfig, index: number) => {
+    if (typeof provider === 'string') {
+      return provider;
+    }
+    return provider.id || index;
+  };
+
+  const handleProviderClick = (provider: string | ProviderConfig) => {
+    if (typeof provider === 'string') {
+      alert('Cannot edit custom providers');
+    } else if (!provider.config) {
+      alert('There is no config for this provider');
+    } else {
+      setSelectedProvider(provider as ProviderConfig);
+    }
+  };
+
+  const handleSave = (config: ProviderConfig['config']) => {
+    if (selectedProvider) {
+      const updatedProviders = providers.map((provider) =>
+        provider.id === selectedProvider.id ? { ...provider, config } : provider,
+      );
+      onChange(updatedProviders);
+      setSelectedProvider(null);
+    }
+  };
+
+  return (
+    <Box mt={2}>
+      <Autocomplete
+        multiple
+        freeSolo
+        options={defaultProviders}
+        value={providers}
+        onChange={(event, newValue: (string | ProviderConfig)[]) => {
+          onChange(newValue.map((value) => (typeof value === 'string' ? { id: value } : value)));
+        }}
+        getOptionLabel={(option) => {
+          if (!option) {
+            return '';
+          }
+          if (typeof option === 'string') {
+            return option;
+          }
+          return (option as ProviderConfig).id || 'Unknown provider';
+        }}
+        renderTags={(value, getTagProps) =>
+          value.map((provider, index: number) => {
+            const label = getProviderLabel(provider);
+            const key = getProviderKey(provider, index);
+
+            return (
+              <Chip
+                variant="outlined"
+                label={label}
+                {...getTagProps({ index })}
+                key={key}
+                onClick={() => handleProviderClick(provider)}
+              />
+            );
+          })
+        }
+        renderInput={(params) => (
+          <TextField
+            {...params}
+            variant="outlined"
+            placeholder="Select LLM providers"
+            helperText={providers.length > 0 ? 'Click a provider to configure its settings.' : null}
+          />
+        )}
+      />
+      {selectedProvider && selectedProvider.id && (
+        <ProviderConfigDialog
+          open={!!selectedProvider}
+          providerId={selectedProvider.id}
+          config={selectedProvider.config}
+          onClose={() => setSelectedProvider(null)}
+          onSave={handleSave}
+        />
+      )}
+    </Box>
+  );
+};
+
+export default ProviderSelector;

package/src/web/nextui/src/app/setup/RunTestSuiteButton.tsx

@@ -0,0 +1,88 @@
+'use client';
+
+import React, { useState } from 'react';
+import { useRouter } from 'next/navigation';
+import { Button, CircularProgress } from '@mui/material';
+
+import { useStore } from '@/util/store';
+import { API_BASE_URL } from '@/util/api';
+
+const RunTestSuiteButton: React.FC = () => {
+  const router = useRouter();
+  const { description, providers, prompts, testCases } = useStore();
+  const [isRunning, setIsRunning] = useState(false);
+  const [progressPercent, setProgressPercent] = useState(0);
+
+  const runTestSuite = async () => {
+    setIsRunning(true);
+
+    const testSuite = {
+      description,
+      providers,
+      prompts,
+      tests: testCases,
+    };
+
+    try {
+      const response = await fetch(`${API_BASE_URL}/api/eval`, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(testSuite),
+      });
+
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+
+      const job = await response.json();
+
+      const intervalId = setInterval(async () => {
+        const progressResponse = await fetch(`${API_BASE_URL}/api/eval/${job.id}`);
+
+        if (!progressResponse.ok) {
+          clearInterval(intervalId);
+          throw new Error(`HTTP error! status: ${progressResponse.status}`);
+        }
+
+        const progressData = await progressResponse.json();
+
+        if (progressData.status === 'completed') {
+          clearInterval(intervalId);
+          setIsRunning(false);
+          router.push('/eval');
+        } else if (progressData.status === 'failed') {
+          clearInterval(intervalId);
+          setIsRunning(false);
+          throw new Error('Job failed');
+        } else {
+          const percent =
+            progressData.total === 0
+              ? 0
+              : Math.round((progressData.progress / progressData.total) * 100);
+          setProgressPercent(percent);
+        }
+      }, 1000);
+    } catch (error) {
+      console.error(error);
+      setIsRunning(false);
+      alert(`An error occurred: ${(error as Error).message}`);
+    }
+  };
+
+  return (
+    <Button variant="contained" color="primary" onClick={runTestSuite} disabled={isRunning}>
+      {isRunning ? (
+        <>
+          <CircularProgress size={24} sx={{ marginRight: 2 }} />
+          {progressPercent.toFixed(0)}% complete
+        </>
+      ) : (
+        'Run Evaluation'
+      )}
+    </Button>
+  );
+};
+
+export default RunTestSuiteButton;

package/src/web/nextui/src/app/setup/TestCaseDialog.tsx

@@ -0,0 +1,108 @@
+import React, { useState } from 'react';
+import {
+  Button,
+  TextField,
+  Box,
+  Dialog,
+  DialogTitle,
+  DialogContent,
+  DialogActions,
+} from '@mui/material';
+import VarsForm from './VarsForm';
+import AssertsForm from './AssertsForm';
+import type { TestCase } from '../../../../../types';
+
+interface TestCaseFormProps {
+  open: boolean;
+  onAdd: (testCase: TestCase, shouldClose: boolean) => void;
+  varsList: string[];
+  initialValues?: TestCase;
+  onCancel: () => void;
+}
+
+const TestCaseForm: React.FC<TestCaseFormProps> = ({
+  open,
+  onAdd,
+  varsList,
+  initialValues,
+  onCancel,
+}) => {
+  const [description, setDescription] = useState(initialValues?.description || '');
+  const [vars, setVars] = useState(initialValues?.vars || {});
+  const [asserts, setAsserts] = useState(initialValues?.assert || []);
+  const [assertsFormKey, setAssertsFormKey] = useState(0);
+
+  React.useEffect(() => {
+    if (initialValues) {
+      setDescription(initialValues.description || '');
+      setVars(initialValues.vars || {});
+      setAsserts(initialValues.assert || []);
+    } else {
+      setDescription('');
+      setVars({});
+      setAsserts([]);
+    }
+  }, [initialValues]);
+
+  const handleAdd = (close: boolean) => {
+    onAdd(
+      {
+        description,
+        vars,
+        assert: asserts,
+      },
+      close,
+    );
+    if (close) {
+      onCancel();
+    }
+    setDescription('');
+    setVars({});
+    setAsserts([]);
+    setAssertsFormKey((prevKey) => prevKey + 1);
+  };
+
+  return (
+    <Dialog open={open} onClose={onCancel} fullWidth maxWidth="md">
+      <DialogTitle>{initialValues ? 'Edit Test Case' : 'Add Test Case'}</DialogTitle>
+      <DialogContent>
+        <Box>
+          {/*
+          <TextField
+            label="Description"
+            value={description}
+            onChange={(e) => setDescription(e.target.value)}
+            fullWidth
+            margin="normal"
+          />
+          */}
+          <VarsForm
+            onAdd={(vars) => setVars(vars)}
+            varsList={varsList}
+            initialValues={initialValues?.vars as Record<string, string>}
+          />
+          <AssertsForm
+            key={assertsFormKey}
+            onAdd={(asserts) => setAsserts(asserts)}
+            initialValues={initialValues?.assert || []}
+          />
+        </Box>
+      </DialogContent>
+      <DialogActions>
+        <Button onClick={handleAdd.bind(this, true)} color="primary" variant="contained">
+          {initialValues ? 'Update Test Case' : 'Add Test Case'}
+        </Button>
+        {!initialValues && (
+          <Button onClick={handleAdd.bind(this, false)} color="primary" variant="contained">
+            Add Another
+          </Button>
+        )}
+        <Button onClick={onCancel} color="secondary">
+          Cancel
+        </Button>
+      </DialogActions>
+    </Dialog>
+  );
+};
+
+export default TestCaseForm;

package/src/web/nextui/src/app/setup/TestCasesSection.tsx

@@ -0,0 +1,154 @@
+import React from 'react';
+import Button from '@mui/material/Button';
+import Copy from '@mui/icons-material/ContentCopy';
+import Delete from '@mui/icons-material/Delete';
+import Edit from '@mui/icons-material/Edit';
+import IconButton from '@mui/material/IconButton';
+import Stack from '@mui/material/Stack';
+import Table from '@mui/material/Table';
+import TableBody from '@mui/material/TableBody';
+import TableCell from '@mui/material/TableCell';
+import TableContainer from '@mui/material/TableContainer';
+import TableHead from '@mui/material/TableHead';
+import TableRow from '@mui/material/TableRow';
+import Typography from '@mui/material/Typography';
+
+import TestCaseDialog from './TestCaseDialog';
+import { useStore } from '../../util/store';
+
+import type { TestCase } from '../../../../../types';
+
+interface TestCasesSectionProps {
+  varsList: string[];
+}
+
+const TestCasesSection: React.FC<TestCasesSectionProps> = ({ varsList }) => {
+  const { testCases, setTestCases } = useStore();
+  const [editingTestCaseIndex, setEditingTestCaseIndex] = React.useState<number | null>(null);
+  const [testCaseDialogOpen, setTestCaseDialogOpen] = React.useState(false);
+
+  const handleAddTestCase = (testCase: TestCase, shouldClose: boolean) => {
+    if (editingTestCaseIndex === null) {
+      setTestCases([...testCases, testCase]);
+    } else {
+      const updatedTestCases = testCases.map((tc, index) =>
+        index === editingTestCaseIndex ? testCase : tc,
+      );
+      setTestCases(updatedTestCases);
+      setEditingTestCaseIndex(null);
+    }
+
+    if (shouldClose) {
+      setTestCaseDialogOpen(false);
+    }
+  };
+
+  const handleRemoveTestCase = (event: React.MouseEvent, index: number) => {
+    event.stopPropagation();
+
+    if (confirm('Are you sure you want to delete this test case?')) {
+      setTestCases(testCases.filter((_, i) => i !== index));
+    }
+  };
+
+  const handleDuplicateTestCase = (event: React.MouseEvent, index: number) => {
+    event.stopPropagation();
+    const duplicatedTestCase = JSON.parse(JSON.stringify(testCases[index]));
+    setTestCases([...testCases, duplicatedTestCase]);
+  };
+
+  return (
+    <>
+      <Stack direction="row" spacing={2} justifyContent="space-between">
+        <Typography variant="h5">Test Cases</Typography>
+        <Button color="primary" onClick={() => setTestCaseDialogOpen(true)} variant="contained">
+          Add Test Case
+        </Button>
+      </Stack>
+      <TableContainer>
+        <Table>
+          <TableHead>
+            <TableRow>
+              <TableCell>Description</TableCell>
+              <TableCell>Assertions</TableCell>
+              <TableCell>Variables</TableCell>
+              <TableCell align="right"></TableCell>
+            </TableRow>
+          </TableHead>
+          <TableBody>
+            {testCases.length === 0 ? (
+              <TableRow>
+                <TableCell colSpan={4} align="center">
+                  No test cases added yet.
+                </TableCell>
+              </TableRow>
+            ) : (
+              testCases.map((testCase, index) => (
+                <TableRow
+                  key={index}
+                  sx={{
+                    '&:hover': {
+                      backgroundColor: 'rgba(0, 0, 0, 0.04)',
+                      cursor: 'pointer',
+                    },
+                  }}
+                  onClick={() => {
+                    setEditingTestCaseIndex(index);
+                    setTestCaseDialogOpen(true);
+                  }}
+                >
+                  <TableCell>
+                    <Typography variant="body2">
+                      {testCase.description || `Test Case #${index + 1}`}
+                    </Typography>
+                  </TableCell>
+                  <TableCell>{testCase.assert?.length || 0} assertions</TableCell>
+                  <TableCell>
+                    {Object.entries(testCase.vars || {})
+                      .map(([k, v]) => k + '=' + v)
+                      .join(', ')}
+                  </TableCell>
+                  <TableCell align="right" sx={{ minWidth: 150 }}>
+                    <IconButton
+                      onClick={() => {
+                        setEditingTestCaseIndex(index);
+                        setTestCaseDialogOpen(true);
+                      }}
+                      size="small"
+                    >
+                      <Edit />
+                    </IconButton>
+                    <IconButton
+                      onClick={(event) => handleDuplicateTestCase(event, index)}
+                      size="small"
+                    >
+                      <Copy />
+                    </IconButton>
+                    <IconButton
+                      onClick={(event) => handleRemoveTestCase(event, index)}
+                      size="small"
+                    >
+                      <Delete />
+                    </IconButton>
+                  </TableCell>
+                </TableRow>
+              ))
+            )}
+          </TableBody>
+        </Table>
+      </TableContainer>
+      <TestCaseDialog
+        open={testCaseDialogOpen}
+        onAdd={handleAddTestCase}
+        varsList={varsList}
+        initialValues={editingTestCaseIndex !== null ? testCases[editingTestCaseIndex] : undefined}
+        onCancel={() => {
+          setEditingTestCaseIndex(null);
+          setTestCaseDialogOpen(false);
+        }}
+      />
+    </>
+  );
+};
+
+export default TestCasesSection;

package/src/web/nextui/src/app/setup/VarsForm.tsx

@@ -0,0 +1,57 @@
+import React, { useEffect } from 'react';
+import { Box, TextField, Typography, Stack } from '@mui/material';
+
+interface VarsFormProps {
+  onAdd: (vars: Record<string, string>) => void;
+  varsList: string[];
+  initialValues?: Record<string, string>;
+}
+
+const VarsForm: React.FC<VarsFormProps> = ({ onAdd, varsList, initialValues }) => {
+  const [vars, setVars] = React.useState<Record<string, string>>(initialValues || {});
+
+  useEffect(() => {
+    const newVars: Record<string, string> = {};
+    varsList.forEach((v) => {
+      newVars[v] = initialValues?.[v] || '';
+    });
+    setVars(newVars);
+  }, [varsList, initialValues]);
+
+  return (
+    <Box my={2}>
+      <Typography variant="h6" mb={2}>
+        Vars
+      </Typography>
+      {varsList.length > 0 ? (
+        <Stack direction="row" spacing={2} alignItems="center">
+          {Object.keys(vars).map((varName, index) => (
+            <Stack key={index} direction="row" spacing={2} alignItems="center">
+              <TextField
+                placeholder={varName}
+                label={varName}
+                value={vars[varName]}
+                fullWidth
+                onChange={(e) => {
+                  const newValue = e.target.value;
+                  const newVars = {
+                    ...vars,
+                    [varName]: newValue,
+                  };
+                  setVars(newVars);
+                  onAdd(newVars);
+                }}
+              />
+            </Stack>
+          ))}
+        </Stack>
+      ) : (
+        <Typography variant="subtitle1" gutterBottom>
+          Add variables to your prompt using the {'{{varname}}'} syntax.
+        </Typography>
+      )}
+    </Box>
+  );
+};
+
+export default VarsForm;

package/src/web/nextui/src/app/setup/page.css

@@ -0,0 +1,3 @@
+.yaml-config {
+  font-size: 10px;
+}