promptfoo 0.20.0 → 0.21.0

package/src/web/nextui/src/app/setup/PromptDialog.tsx

@@ -1,77 +0,0 @@
-import React from 'react';
-import {
-  Dialog,
-  DialogTitle,
-  DialogContent,
-  DialogActions,
-  TextField,
-  Button,
-} from '@mui/material';
-
-interface PromptDialogProps {
-  open: boolean;
-  prompt: string;
-  index: number;
-  onAdd: (prompt: string) => void;
-  onCancel: () => void;
-}
-
-const PromptDialog: React.FC<PromptDialogProps> = ({ open, prompt, index, onAdd, onCancel }) => {
-  const [editingPrompt, setEditingPrompt] = React.useState(prompt);
-  const textFieldRef = React.useRef<HTMLInputElement>(null);
-
-  React.useEffect(() => {
-    setEditingPrompt(prompt);
-  }, [prompt]);
-
-  const handleAdd = (close: boolean) => {
-    onAdd(editingPrompt);
-    setEditingPrompt('');
-    if (close) {
-      onCancel();
-    } else if (textFieldRef.current) {
-      textFieldRef.current.focus();
-    }
-  };
-
-  return (
-    <Dialog open={open} onClose={onCancel} fullWidth maxWidth="md">
-      <DialogTitle>{`Edit Prompt ${index + 1}`}</DialogTitle>
-      <DialogContent>
-        <TextField
-          value={editingPrompt}
-          onChange={(e) => setEditingPrompt(e.target.value)}
-          fullWidth
-          margin="normal"
-          multiline
-          placeholder="The quick brown {{animal1}} jumps over the lazy {{animal2}}."
-          helperText="Tip: use the {{varname}} syntax to add variables to your prompt."
-          inputRef={textFieldRef}
-        />
-      </DialogContent>
-      <DialogActions>
-        <Button
-          onClick={handleAdd.bind(null, true)}
-          color="primary"
-          variant="contained"
-          disabled={!editingPrompt.length}
-        >
-          Add
-        </Button>
-        <Button
-          onClick={handleAdd.bind(null, false)}
-          color="primary"
-          variant="contained"
-          disabled={!editingPrompt.length}
-        >
-          Add Another
-        </Button>
-        <Button onClick={onCancel} color="secondary">
-          Cancel
-        </Button>
-      </DialogActions>
-    </Dialog>
-  );
-};
-
-export default PromptDialog;

package/src/web/nextui/src/app/setup/PromptsSection.tsx

@@ -1,190 +0,0 @@
-import React, { useState, useRef, useEffect } from 'react';
-import Button from '@mui/material/Button';
-import Typography from '@mui/material/Typography';
-import IconButton from '@mui/material/IconButton';
-import Table from '@mui/material/Table';
-import TableBody from '@mui/material/TableBody';
-import TableCell from '@mui/material/TableCell';
-import TableContainer from '@mui/material/TableContainer';
-import TableRow from '@mui/material/TableRow';
-import Tooltip from '@mui/material/Tooltip';
-import Stack from '@mui/material/Stack';
-import Edit from '@mui/icons-material/Edit';
-import Delete from '@mui/icons-material/Delete';
-import Publish from '@mui/icons-material/Publish';
-import Copy from '@mui/icons-material/ContentCopy';
-
-import PromptDialog from './PromptDialog';
-import { useStore } from '../../util/store';
-
-const PromptsSection: React.FC = () => {
-  const [promptDialogOpen, setPromptDialogOpen] = useState(false);
-  const [editingPromptIndex, setEditingPromptIndex] = useState<number | null>(null);
-
-  const { prompts, setPrompts } = useStore();
-  const newPromptInputRef = useRef<HTMLInputElement>(null);
-
-  useEffect(() => {
-    if (editingPromptIndex !== null && editingPromptIndex > 0 && newPromptInputRef.current) {
-      newPromptInputRef.current.focus();
-    }
-  }, [editingPromptIndex]);
-
-  const handleEditPrompt = (index: number) => {
-    setEditingPromptIndex(index);
-    setPromptDialogOpen(true);
-  };
-
-  const handleAddPromptFromFile = (event: React.ChangeEvent<HTMLInputElement>) => {
-    event.stopPropagation();
-    event.preventDefault();
-
-    const file = event.target.files?.[0];
-    if (file) {
-      const reader = new FileReader();
-      reader.onload = (e) => {
-        const text = e.target?.result?.toString();
-        if (text) {
-          setPrompts([...prompts, text]);
-        }
-      };
-      reader.readAsText(file);
-    }
-  };
-
-  const handleDuplicatePrompt = (event: React.MouseEvent, index: number) => {
-    event.stopPropagation();
-    const duplicatedPrompt = prompts[index];
-    setPrompts([...prompts, duplicatedPrompt]);
-  };
-
-  const handleChangePrompt = (index: number, newPrompt: string) => {
-    setPrompts(prompts.map((p, i) => (i === index ? newPrompt : p)));
-  };
-
-  const handleRemovePrompt = (event: React.MouseEvent, indexToRemove: number) => {
-    event.stopPropagation();
-
-    if (confirm('Are you sure you want to remove this prompt?')) {
-      setPrompts(prompts.filter((_, index) => index !== indexToRemove));
-    }
-  };
-
-  return (
-    <div>
-      <Stack direction="row" spacing={2} justifyContent="space-between">
-        <Typography variant="h5">Prompts</Typography>
-        <div>
-          <label htmlFor={`file-input-add-prompt`}>
-            <Tooltip title="Upload prompt from file">
-              <span>
-                <IconButton component="span">
-                  <Publish />
-                </IconButton>
-                <input
-                  id={`file-input-add-prompt`}
-                  type="file"
-                  accept=".txt,.md"
-                  onChange={handleAddPromptFromFile}
-                  style={{ display: 'none' }}
-                />
-              </span>
-            </Tooltip>
-          </label>
-          <Button
-            color="primary"
-            onClick={() => {
-              setPromptDialogOpen(true);
-            }}
-            variant="contained"
-          >
-            Add Prompt
-          </Button>
-        </div>
-      </Stack>
-      <TableContainer>
-        <Table>
-          <TableBody>
-            {prompts.length === 0 ? (
-              <TableRow>
-                <TableCell colSpan={2} align="center">
-                  No prompts added yet.
-                </TableCell>
-              </TableRow>
-            ) : (
-              prompts.map((prompt, index) => (
-                <TableRow
-                  key={index}
-                  sx={{
-                    '&:hover': {
-                      backgroundColor: 'rgba(0, 0, 0, 0.04)',
-                      cursor: 'pointer',
-                    },
-                  }}
-                  onClick={() => handleEditPrompt(index)}
-                >
-                  <TableCell>
-                    <Typography variant="body2">
-                      {`Prompt #${index + 1}: `}
-                      {(prompt.length > 250 ? prompt.slice(0, 250) + ' ...' : prompt)
-                        .split(/({{\w+}})/g)
-                        .map((part, i) =>
-                          /{{\w+}}/g.test(part) ? (
-                            <span
-                              key={i}
-                              style={{
-                                backgroundColor: 'linen',
-                                padding: '0.25rem',
-                                borderRadius: '4px',
-                              }}
-                            >
-                              {part}
-                            </span>
-                          ) : (
-                            part
-                          ),
-                        )}
-                    </Typography>
-                  </TableCell>
-                  <TableCell align="right" sx={{ minWidth: 150 }}>
-                    <IconButton onClick={() => handleEditPrompt(index)} size="small">
-                      <Edit />
-                    </IconButton>
-                    <IconButton
-                      onClick={(event) => handleDuplicatePrompt(event, index)}
-                      size="small"
-                    >
-                      <Copy />
-                    </IconButton>
-                    <IconButton onClick={(event) => handleRemovePrompt(event, index)} size="small">
-                      <Delete />
-                    </IconButton>
-                  </TableCell>
-                </TableRow>
-              ))
-            )}
-          </TableBody>
-        </Table>
-      </TableContainer>
-      <PromptDialog
-        open={promptDialogOpen}
-        prompt={editingPromptIndex !== null ? prompts[editingPromptIndex] : ''}
-        index={editingPromptIndex !== null ? editingPromptIndex : 0}
-        onAdd={(newPrompt) => {
-          if (editingPromptIndex !== null) {
-            handleChangePrompt(editingPromptIndex, newPrompt);
-          } else {
-            setPrompts([...prompts, newPrompt]);
-          }
-          setEditingPromptIndex(null);
-        }}
-        onCancel={() => {
-          setEditingPromptIndex(null);
-          setPromptDialogOpen(false);
-        }}
-      />
-    </div>
-  );
-};
-
-export default PromptsSection;

package/src/web/nextui/src/app/setup/ProviderConfigDialog.tsx

@@ -1,99 +0,0 @@
-import React from 'react';
-import {
-  Box,
-  Dialog,
-  DialogTitle,
-  DialogContent,
-  TextField,
-  DialogActions,
-  Button,
-} from '@mui/material';
-import { ProviderOptions } from '../../../../../types';
-
-interface ProviderConfigDialogProps {
-  open: boolean;
-  providerId: string;
-  config: ProviderOptions['config'];
-  onClose: () => void;
-  onSave: (config: ProviderOptions['config']) => void;
-}
-
-const ProviderConfigDialog: React.FC<ProviderConfigDialogProps> = ({
-  open,
-  providerId,
-  config,
-  onClose,
-  onSave,
-}) => {
-  const [localConfig, setLocalConfig] = React.useState(config);
-
-  React.useEffect(() => {
-    setLocalConfig(config);
-  }, [config]);
-
-  const handleSave = () => {
-    onSave(localConfig);
-  };
-
-  return (
-    <Dialog open={open} onClose={onClose}>
-      <DialogTitle>Edit {providerId}</DialogTitle>
-      <DialogContent>
-        {Object.keys(localConfig).map((key) => {
-          const value = localConfig[key];
-          let handleChange;
-
-          if (
-            typeof value === 'number' ||
-            typeof value === 'boolean' ||
-            typeof value === 'string'
-          ) {
-            if (typeof value === 'number') {
-              handleChange = (e: React.ChangeEvent<HTMLInputElement>) =>
-                setLocalConfig({ ...localConfig, [key]: parseFloat(e.target.value) });
-            } else if (typeof value === 'boolean') {
-              handleChange = (e: React.ChangeEvent<HTMLInputElement>) =>
-                setLocalConfig({ ...localConfig, [key]: e.target.value === 'true' });
-            } else {
-              handleChange = (e: React.ChangeEvent<HTMLInputElement>) =>
-                setLocalConfig({ ...localConfig, [key]: e.target.value });
-            }
-
-            return (
-              <Box key={key} my={2}>
-                <TextField
-                  label={key}
-                  value={value}
-                  onChange={handleChange}
-                  fullWidth
-                  type={typeof value === 'number' ? 'number' : 'text'}
-                />
-              </Box>
-            );
-          } else {
-            return (
-              <Box key={key} my={2}>
-                <TextField
-                  label={key}
-                  value={JSON.stringify(value)}
-                  onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
-                    setLocalConfig({ ...localConfig, [key]: JSON.parse(e.target.value) })
-                  }
-                  fullWidth
-                  multiline
-                  minRows={3}
-                />
-              </Box>
-            );
-          }
-        })}
-      </DialogContent>
-      <DialogActions>
-        <Button onClick={onClose}>Cancel</Button>
-        <Button onClick={handleSave}>Save</Button>
-      </DialogActions>
-    </Dialog>
-  );
-};
-
-export default ProviderConfigDialog;

package/src/web/nextui/src/app/setup/ProviderSelector.tsx

@@ -1,149 +0,0 @@
-import React from 'react';
-import { Autocomplete, Box, Chip, TextField } from '@mui/material';
-import { ProviderOptions } from '../../../../../types';
-import ProviderConfigDialog from './ProviderConfigDialog';
-
-const defaultProviders: ProviderOptions[] = [
-  {
-    id: 'replicate:replicate/llama70b-v2-chat:e951f18578850b652510200860fc4ea62b3b16fac280f83ff32282f87bbd2e48',
-    config: { temperature: 0.5 },
-  },
-]
-  .concat(
-    [
-      'anthropic:claude-1',
-      'anthropic:claude-1-100k',
-      'anthropic:claude-instant-1',
-      'anthropic:claude-instant-1-100k',
-    ].map((id) => ({ id, config: { temperature: 0.5 } })),
-  )
-  .concat(
-    [
-      'openai:gpt-3.5-turbo',
-      'openai:gpt-3.5-turbo-0301',
-      'openai:gpt-3.5-turbo-0613',
-      'openai:gpt-3.5-turbo-16k',
-      'openai:gpt-3.5-turbo-16k-0613',
-      'openai:gpt-4',
-      'openai:gpt-4-0314',
-      'openai:gpt-4-0613',
-      'openai:gpt-4-32k',
-      'openai:gpt-4-32k-0314',
-    ].map((id) => ({ id, config: { temperature: 0.5, max_tokens: 1024 } })),
-  )
-  .concat(
-    [
-      'azureopenai:gpt-3.5-turbo',
-      'azureopenai:gpt-3.5-turbo-0301',
-      'azureopenai:gpt-3.5-turbo-0613',
-      'azureopenai:gpt-3.5-turbo-16k',
-      'azureopenai:gpt-3.5-turbo-16k-0613',
-      'azureopenai:gpt-4',
-      'azureopenai:gpt-4-0314',
-      'azureopenai:gpt-4-0613',
-      'azureopenai:gpt-4-32k',
-      'azureopenai:gpt-4-32k-0314',
-    ].map((id) => ({ id, config: { temperature: 0.5, max_tokens: 1024 } })),
-  )
-  .sort((a, b) => a.id.localeCompare(b.id));
-
-interface ProviderSelectorProps {
-  providers: ProviderOptions[];
-  onChange: (providers: ProviderOptions[]) => void;
-}
-
-const ProviderSelector: React.FC<ProviderSelectorProps> = ({ providers, onChange }) => {
-  const [selectedProvider, setSelectedProvider] = React.useState<ProviderOptions | null>(null);
-
-  const getProviderLabel = (provider: string | ProviderOptions) => {
-    if (typeof provider === 'string') {
-      return provider;
-    }
-    return provider.id || 'Unknown provider';
-  };
-
-  const getProviderKey = (provider: string | ProviderOptions, index: number) => {
-    if (typeof provider === 'string') {
-      return provider;
-    }
-    return provider.id || index;
-  };
-
-  const handleProviderClick = (provider: string | ProviderOptions) => {
-    if (typeof provider === 'string') {
-      alert('Cannot edit custom providers');
-    } else if (!provider.config) {
-      alert('There is no config for this provider');
-    } else {
-      setSelectedProvider(provider as ProviderOptions);
-    }
-  };
-
-  const handleSave = (config: ProviderOptions['config']) => {
-    if (selectedProvider) {
-      const updatedProviders = providers.map((provider) =>
-        provider.id === selectedProvider.id ? { ...provider, config } : provider,
-      );
-      onChange(updatedProviders);
-      setSelectedProvider(null);
-    }
-  };
-
-  return (
-    <Box mt={2}>
-      <Autocomplete
-        multiple
-        freeSolo
-        options={defaultProviders}
-        value={providers}
-        onChange={(event, newValue: (string | ProviderOptions)[]) => {
-          onChange(newValue.map((value) => (typeof value === 'string' ? { id: value } : value)));
-        }}
-        getOptionLabel={(option) => {
-          if (!option) {
-            return '';
-          }
-          if (typeof option === 'string') {
-            return option;
-          }
-          return (option as ProviderOptions).id || 'Unknown provider';
-        }}
-        renderTags={(value, getTagProps) =>
-          value.map((provider, index: number) => {
-            const label = getProviderLabel(provider);
-            const key = getProviderKey(provider, index);
-
-            return (
-              <Chip
-                variant="outlined"
-                label={label}
-                {...getTagProps({ index })}
-                key={key}
-                onClick={() => handleProviderClick(provider)}
-              />
-            );
-          })
-        }
-        renderInput={(params) => (
-          <TextField
-            {...params}
-            variant="outlined"
-            placeholder="Select LLM providers"
-            helperText={providers.length > 0 ? 'Click a provider to configure its settings.' : null}
-          />
-        )}
-      />
-      {selectedProvider && selectedProvider.id && (
-        <ProviderConfigDialog
-          open={!!selectedProvider}
-          providerId={selectedProvider.id}
-          config={selectedProvider.config}
-          onClose={() => setSelectedProvider(null)}
-          onSave={handleSave}
-        />
-      )}
-    </Box>
-  );
-};
-
-export default ProviderSelector;

package/src/web/nextui/src/app/setup/RunTestSuiteButton.tsx

@@ -1,88 +0,0 @@
-'use client';
-
-import React, { useState } from 'react';
-import { useRouter } from 'next/navigation';
-import { Button, CircularProgress } from '@mui/material';
-
-import { useStore } from '@/util/store';
-import { API_BASE_URL } from '@/util/api';
-
-const RunTestSuiteButton: React.FC = () => {
-  const router = useRouter();
-  const { description, providers, prompts, testCases } = useStore();
-  const [isRunning, setIsRunning] = useState(false);
-  const [progressPercent, setProgressPercent] = useState(0);
-
-  const runTestSuite = async () => {
-    setIsRunning(true);
-
-    const testSuite = {
-      description,
-      providers,
-      prompts,
-      tests: testCases,
-    };
-
-    try {
-      const response = await fetch(`${API_BASE_URL}/api/eval`, {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify(testSuite),
-      });
-
-      if (!response.ok) {
-        throw new Error(`HTTP error! status: ${response.status}`);
-      }
-
-      const job = await response.json();
-
-      const intervalId = setInterval(async () => {
-        const progressResponse = await fetch(`${API_BASE_URL}/api/eval/${job.id}`);
-
-        if (!progressResponse.ok) {
-          clearInterval(intervalId);
-          throw new Error(`HTTP error! status: ${progressResponse.status}`);
-        }
-
-        const progressData = await progressResponse.json();
-
-        if (progressData.status === 'completed') {
-          clearInterval(intervalId);
-          setIsRunning(false);
-          router.push('/eval');
-        } else if (progressData.status === 'failed') {
-          clearInterval(intervalId);
-          setIsRunning(false);
-          throw new Error('Job failed');
-        } else {
-          const percent =
-            progressData.total === 0
-              ? 0
-              : Math.round((progressData.progress / progressData.total) * 100);
-          setProgressPercent(percent);
-        }
-      }, 1000);
-    } catch (error) {
-      console.error(error);
-      setIsRunning(false);
-      alert(`An error occurred: ${(error as Error).message}`);
-    }
-  };
-
-  return (
-    <Button variant="contained" color="primary" onClick={runTestSuite} disabled={isRunning}>
-      {isRunning ? (
-        <>
-          <CircularProgress size={24} sx={{ marginRight: 2 }} />
-          {progressPercent.toFixed(0)}% complete
-        </>
-      ) : (
-        'Run Evaluation'
-      )}
-    </Button>
-  );
-};
-
-export default RunTestSuiteButton;