promptfoo 0.20.0 → 0.21.0

package/src/web/nextui/src/app/eval/ResultsView.tsx

@@ -1,301 +0,0 @@
-import * as React from 'react';
-
-import invariant from 'tiny-invariant';
-import Button from '@mui/material/Button';
-import Box from '@mui/material/Box';
-import Checkbox from '@mui/material/Checkbox';
-import CircularProgress from '@mui/material/CircularProgress';
-import FormControl from '@mui/material/FormControl';
-import FormControlLabel from '@mui/material/FormControlLabel';
-import InputLabel from '@mui/material/InputLabel';
-import ListItemText from '@mui/material/ListItemText';
-import MenuItem from '@mui/material/MenuItem';
-import OutlinedInput from '@mui/material/OutlinedInput';
-import Paper from '@mui/material/Box';
-import Select, { SelectChangeEvent } from '@mui/material/Select';
-import Slider from '@mui/material/Slider';
-import Stack from '@mui/material/Stack';
-import Tooltip from '@mui/material/Tooltip';
-import Typography from '@mui/material/Typography';
-import ShareIcon from '@mui/icons-material/Share';
-import VisibilityIcon from '@mui/icons-material/Visibility';
-import { styled } from '@mui/system';
-
-import ResultsCharts from './ResultsCharts';
-import ResultsTable from './ResultsTable';
-import ConfigModal from './ConfigModal';
-import ShareModal from './ShareModal';
-import { useStore } from './store';
-
-import type { VisibilityState } from '@tanstack/table-core';
-import type { FilterMode } from './types';
-
-const ResponsiveStack = styled(Stack)(({ theme }) => ({
-  maxWidth: '100%',
-  flexWrap: 'wrap',
-  [theme.breakpoints.down('sm')]: {
-    flexDirection: 'column',
-  },
-}));
-
-function filenameToDate(filename: string) {
-  const dateString = filename.slice('eval-'.length, filename.length - '.json'.length);
-
-  // Replace hyphens with colons where necessary (Windows compatibility).
-  const dateParts = dateString.split('T');
-  const timePart = dateParts[1].replace(/-/g, ':');
-  const formattedDateString = `${dateParts[0]}T${timePart}`;
-
-  const date = new Date(formattedDateString);
-  return date.toLocaleDateString('en-US', {
-    year: 'numeric',
-    month: 'long',
-    day: 'numeric',
-    hour: '2-digit',
-    minute: '2-digit',
-    second: '2-digit',
-    timeZoneName: 'short',
-  });
-}
-
-interface ResultsViewProps {
-  recentFiles: string[];
-  onRecentFileSelected: (file: string) => void;
-}
-
-export default function ResultsView({ recentFiles, onRecentFileSelected }: ResultsViewProps) {
-  const { table, config } = useStore();
-  const [maxTextLength, setMaxTextLength] = React.useState(250);
-  const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({});
-  const [selectedColumns, setSelectedColumns] = React.useState<string[]>([]);
-
-  const [failureFilter, setFailureFilter] = React.useState<{ [key: string]: boolean }>({});
-  const handleFailureFilterToggle = (columnId: string, checked: boolean) => {
-    setFailureFilter((prevFailureFilter) => ({ ...prevFailureFilter, [columnId]: checked }));
-  };
-
-  const [filterMode, setFilterMode] = React.useState<FilterMode>('all');
-  const handleFilterModeChange = (event: SelectChangeEvent<unknown>) => {
-    const mode = event.target.value as FilterMode;
-    setFilterMode(mode);
-
-    const newFailureFilter: { [key: string]: boolean } = {};
-    head.prompts.forEach((_, idx) => {
-      const columnId = `Prompt ${idx + 1}`;
-      newFailureFilter[columnId] = mode === 'failures';
-    });
-    setFailureFilter(newFailureFilter);
-  };
-
-  const [wordBreak, setWordBreak] = React.useState<'break-word' | 'break-all'>('break-word');
-  const handleWordBreakChange = (event: React.ChangeEvent<HTMLInputElement>) => {
-    setWordBreak(event.target.checked ? 'break-all' : 'break-word');
-  };
-
-  const [shareModalOpen, setShareModalOpen] = React.useState(false);
-  const [shareUrl, setShareUrl] = React.useState('');
-  const [shareLoading, setShareLoading] = React.useState(false);
-
-  const handleShareButtonClick = async () => {
-    setShareLoading(true);
-    try {
-      const response = await fetch('https://api.promptfoo.dev/eval', {
-        method: 'POST',
-        headers: {
-          'Content-Type': 'application/json',
-        },
-        body: JSON.stringify({
-          data: {
-            version: 1,
-            results: {
-              table,
-            },
-            config,
-          },
-        }),
-      });
-
-      const { id } = await response.json();
-      const shareUrl = `https://app.promptfoo.dev/eval/${id}`;
-      setShareUrl(shareUrl);
-      setShareModalOpen(true);
-    } catch {
-      alert('Sorry, something went wrong.');
-    } finally {
-      setShareLoading(false);
-    }
-  };
-
-  const [configModalOpen, setConfigModalOpen] = React.useState(false);
-
-  invariant(table, 'Table data must be loaded before rendering ResultsView');
-  const { head } = table;
-
-  const handleChange = (event: SelectChangeEvent<typeof selectedColumns>) => {
-    const {
-      target: { value },
-    } = event;
-    setSelectedColumns(typeof value === 'string' ? value.split(',') : value);
-
-    const allColumns = [
-      ...head.vars.map((_, idx) => `Variable ${idx + 1}`),
-      ...head.prompts.map((_, idx) => `Prompt ${idx + 1}`),
-    ];
-    const newColumnVisibility: VisibilityState = {};
-    allColumns.forEach((col) => {
-      newColumnVisibility[col] = (typeof value === 'string' ? value.split(',') : value).includes(
-        col,
-      );
-    });
-    setColumnVisibility(newColumnVisibility);
-  };
-
-  const columnData = [
-    ...head.vars.map((_, idx) => ({
-      value: `Variable ${idx + 1}`,
-      label: `Variable ${idx + 1}`,
-      group: 'Variables',
-    })),
-    ...head.prompts.map((_, idx) => ({
-      value: `Prompt ${idx + 1}`,
-      label: `Prompt ${idx + 1}`,
-      group: 'Prompts',
-    })),
-  ];
-
-  // Set all columns as selected by default
-  React.useEffect(() => {
-    setSelectedColumns([
-      ...head.vars.map((_, idx) => `Variable ${idx + 1}`),
-      ...head.prompts.map((_, idx) => `Prompt ${idx + 1}`),
-    ]);
-  }, [head]);
-
-  return (
-    <div style={{ marginLeft: '1rem', marginRight: '1rem' }}>
-      <Paper py="md">
-        <ResponsiveStack direction="row" spacing={4} alignItems="center">
-          <Box>
-            {recentFiles && recentFiles.length > 0 && (
-              <FormControl sx={{ m: 1, minWidth: 200 }} size="small">
-                <InputLabel>View run</InputLabel>
-                <Select
-                  key={recentFiles.join(',')}
-                  className="recent-files"
-                  label="Previous runs"
-                  defaultValue={recentFiles[0]}
-                  onChange={(e: SelectChangeEvent) => onRecentFileSelected(e.target.value)}
-                >
-                  {recentFiles.map((file) => (
-                    <MenuItem key={file} value={file}>
-                      {filenameToDate(file)}
-                    </MenuItem>
-                  ))}
-                </Select>
-              </FormControl>
-            )}
-          </Box>
-          <Box>
-            <FormControl sx={{ m: 1, minWidth: 200 }} size="small">
-              <InputLabel id="visible-columns-label">Show columns</InputLabel>
-              <Select
-                labelId="visible-columns-label"
-                id="visible-columns"
-                multiple
-                value={selectedColumns}
-                onChange={handleChange}
-                input={<OutlinedInput label="Visible columns" />}
-                renderValue={(selected: string[]) => selected.join(', ')}
-              >
-                {columnData.map((column) => (
-                  <MenuItem dense key={column.value} value={column.value}>
-                    <Checkbox checked={selectedColumns.indexOf(column.value) > -1} />
-                    <ListItemText primary={column.label} />
-                  </MenuItem>
-                ))}
-              </Select>
-            </FormControl>
-          </Box>
-          <Box>
-            <FormControl sx={{ minWidth: 180 }} size="small">
-              <InputLabel id="failure-filter-mode-label">Filter</InputLabel>
-              <Select
-                labelId="filter-mode-label"
-                id="filter-mode"
-                value={filterMode}
-                onChange={handleFilterModeChange}
-                label="Filter"
-              >
-                <MenuItem value="all">Show all results</MenuItem>
-                <MenuItem value="failures">Show failures only</MenuItem>
-                <MenuItem value="different">Show different only</MenuItem>
-              </Select>
-            </FormControl>
-          </Box>
-          <Box>
-            <Typography mt={2}>Max text length: {maxTextLength}</Typography>
-            <Slider
-              min={25}
-              max={1000}
-              value={maxTextLength}
-              onChange={(_, val: number | number[]) => setMaxTextLength(val as number)}
-            />
-          </Box>
-          <Box>
-            <Tooltip title="Forcing line breaks makes it easier to adjust column widths to your liking">
-              <FormControlLabel
-                control={
-                  <Checkbox checked={wordBreak === 'break-all'} onChange={handleWordBreakChange} />
-                }
-                label="Force line breaks"
-              />
-            </Tooltip>
-          </Box>
-          <Box flexGrow={1} />
-          <Box display="flex" justifyContent="flex-end">
-            <ResponsiveStack direction="row" spacing={2}>
-              {config && (
-                <Tooltip title="View config">
-                  <Button
-                    color="primary"
-                    onClick={() => setConfigModalOpen(true)}
-                    startIcon={<VisibilityIcon />}
-                  >
-                    Config
-                  </Button>
-                </Tooltip>
-              )}
-              {config?.sharing && (
-                <Tooltip title="Generate a unique URL that others can access">
-                  <Button
-                    color="primary"
-                    onClick={handleShareButtonClick}
-                    disabled={shareLoading}
-                    startIcon={shareLoading ? <CircularProgress size={16} /> : <ShareIcon />}
-                  >
-                    Share
-                  </Button>
-                </Tooltip>
-              )}
-            </ResponsiveStack>
-          </Box>
-        </ResponsiveStack>
-      </Paper>
-      <ResultsCharts columnVisibility={columnVisibility} />
-      <ResultsTable
-        maxTextLength={maxTextLength}
-        columnVisibility={columnVisibility}
-        wordBreak={wordBreak}
-        filterMode={filterMode}
-        failureFilter={failureFilter}
-        onFailureFilterToggle={handleFailureFilterToggle}
-      />
-      <ConfigModal open={configModalOpen} onClose={() => setConfigModalOpen(false)} />
-      <ShareModal
-        open={shareModalOpen}
-        onClose={() => setShareModalOpen(false)}
-        shareUrl={shareUrl}
-      />
-    </div>
-  );
-}

package/src/web/nextui/src/app/eval/ShareModal.tsx

@@ -1,70 +0,0 @@
-import React, { useRef, useState } from 'react';
-import Dialog from '@mui/material/Dialog';
-import DialogTitle from '@mui/material/DialogTitle';
-import DialogContent from '@mui/material/DialogContent';
-import DialogContentText from '@mui/material/DialogContentText';
-import DialogActions from '@mui/material/DialogActions';
-import Button from '@mui/material/Button';
-import TextField from '@mui/material/TextField';
-import IconButton from '@mui/material/IconButton';
-import FileCopyIcon from '@mui/icons-material/FileCopy';
-import CheckIcon from '@mui/icons-material/Check';
-
-interface ShareModalProps {
-  open: boolean;
-  onClose: () => void;
-  shareUrl: string;
-}
-
-const ShareModal: React.FC<ShareModalProps> = ({ open, onClose, shareUrl }) => {
-  const inputRef = useRef<HTMLInputElement>(null);
-  const [copied, setCopied] = useState(false);
-
-  const handleCopyClick = () => {
-    if (inputRef.current) {
-      inputRef.current.select();
-      document.execCommand('copy');
-      setCopied(true);
-    }
-  };
-
-  const handleClose = () => {
-    onClose();
-    setCopied(false);
-  };
-
-  return (
-    <Dialog
-      open={open}
-      onClose={handleClose}
-      PaperProps={{ style: { minWidth: 'min(660px, 100%)' } }}
-    >
-      <DialogTitle>Your eval is ready to share</DialogTitle>
-      <DialogContent>
-        <TextField
-          inputRef={inputRef}
-          value={shareUrl}
-          fullWidth
-          InputProps={{
-            readOnly: true,
-            endAdornment: (
-              <IconButton onClick={handleCopyClick}>
-                {copied ? <CheckIcon /> : <FileCopyIcon />}
-              </IconButton>
-            ),
-          }}
-        />
-        <DialogContentText sx={{ fontSize: '0.75rem' }}>
-          Shared URLs are deleted after 1 week.
-        </DialogContentText>
-      </DialogContent>
-      <DialogActions>
-        <Button onClick={handleClose} color="primary">
-          Close
-        </Button>
-      </DialogActions>
-    </Dialog>
-  );
-};
-
-export default ShareModal;

package/src/web/nextui/src/app/eval/[id]/not-found.tsx

@@ -1,5 +0,0 @@
-import './page.css';
-
-export default function NotFound() {
-  return <div className="error">Eval not found</div>;
-}

package/src/web/nextui/src/app/eval/[id]/page.css

@@ -1,9 +0,0 @@
-.error {
-  display: flex;
-  flex-direction: column;
-  gap: 1.5rem;
-  justify-content: center;
-  align-items: center;
-  height: 9rem;
-  font-size: 1.5rem;
-}

package/src/web/nextui/src/app/eval/[id]/page.tsx

@@ -1,20 +0,0 @@
-import React from 'react';
-import { notFound } from 'next/navigation';
-
-import Eval from '../Eval';
-
-import './page.css';
-
-export async function generateStaticParams() {
-  return [];
-}
-
-export default async function Page({ params }: { params: { id: string } }) {
-  const response = await fetch(`https://api.promptfoo.dev/eval/${params.id}`);
-  if (!response.ok) {
-    notFound();
-  }
-  const data = await response.json();
-
-  return <Eval preloadedData={data} />;
-}

package/src/web/nextui/src/app/eval/page.tsx

@@ -1,8 +0,0 @@
-import React from 'react';
-
-import Eval from './Eval';
-
-export default function Page() {
-  // Just return local eval from websocket
-  return <Eval />;
-}

package/src/web/nextui/src/app/eval/store.ts

@@ -1,18 +0,0 @@
-import { create } from 'zustand';
-
-import type { EvalTable, UnifiedConfig } from './types';
-
-interface TableState {
-  table: EvalTable | null;
-  setTable: (table: EvalTable | null) => void;
-
-  config: Partial<UnifiedConfig> | null;
-  setConfig: (config: Partial<UnifiedConfig> | null) => void;
-}
-
-export const useStore = create<TableState>((set) => ({
-  table: null,
-  setTable: (table: EvalTable | null) => set(() => ({ table })),
-  config: null,
-  setConfig: (config: Partial<UnifiedConfig> | null) => set(() => ({ config })),
-}));

package/src/web/nextui/src/app/eval/types.ts

@@ -1,20 +0,0 @@
-import type { Prompt, EvaluateTableOutput } from '../../../../../types';
-
-export type EvalHead = {
-  prompts: Prompt[];
-  vars: string[];
-};
-
-export type EvalRow = {
-  outputs: EvaluateTableOutput[];
-  vars: string[]; // model outputs
-};
-
-export type EvalTable = {
-  head: EvalHead;
-  body: EvalRow[];
-};
-
-export type FilterMode = 'all' | 'failures' | 'different';
-
-export * from '../../../../../types';

package/src/web/nextui/src/app/globals.css

@@ -1,58 +0,0 @@
-/* This CSS is common to all pages */
-
-* {
-  box-sizing: border-box;
-}
-
-:root {
-  font-synthesis: none;
-  text-rendering: optimizeLegibility;
-  -webkit-font-smoothing: antialiased;
-  -moz-osx-font-smoothing: grayscale;
-  -webkit-text-size-adjust: 100%;
-
-  /* Light mode colors */
-  --background-color: #ffffff;
-  --text-color: #404040;
-  --border-color: lightgray;
-  --table-border-color: lightgray;
-  --pass-color: green;
-  --fail-color: #ad0000;
-  --smalltext-color: gray;
-  --success-background-color: #d1ffd7;
-  --variable-background-color: #f7f7f7;
-  --header-background-color: #fffdf7;
-  --insert-highlight-color: #d4fcbc;
-  --delete-highlight-color: #fbb6c2;
-}
-
-/* Dark mode colors */
-[data-theme='dark'] {
-  /* Keep synced with prefers-color-scheme above */
-  --background-color: #1a1a1a;
-  --text-color: #f0f0f0;
-  --border-color: #444444;
-  --table-border-color: #444444;
-  --pass-color: #4caf50;
-  --fail-color: #f44336;
-  --smalltext-color: #888888;
-  --success-background-color: #216d2b;
-  --variable-background-color: #333;
-  --header-background-color: #333;
-  --insert-highlight-color: #4f8a34;
-  --delete-highlight-color: #8a3434;
-}
-
-html {
-  font-size: 16px;
-  background-color: var(--background-color);
-  color: var(--text-color);
-}
-
-body {
-  margin: 0;
-}
-
-* {
-  box-sizing: border-box;
-}

package/src/web/nextui/src/app/layout.tsx

@@ -1,25 +0,0 @@
-import './globals.css';
-import type { Metadata } from 'next';
-import { Roboto } from 'next/font/google';
-import { PageShell } from './components/PageShell';
-
-const roboto = Roboto({
-  weight: ['400', '500', '700'],
-  style: ['normal'],
-  subsets: ['latin'],
-});
-
-export const metadata: Metadata = {
-  title: 'promptfoo',
-  description: 'LLM testing and evaluation',
-};
-
-export default function RootLayout({ children }: { children: React.ReactNode }) {
-  return (
-    <html lang="en">
-      <body className={roboto.className}>
-        <PageShell>{children}</PageShell>
-      </body>
-    </html>
-  );
-}

package/src/web/nextui/src/app/page.tsx

@@ -1,7 +0,0 @@
-import { redirect } from 'next/navigation';
-
-import './Home.css';
-
-export default function Page() {
-  redirect('/eval');
-}

package/src/web/nextui/src/app/setup/AssertsForm.tsx

@@ -1,118 +0,0 @@
-// src/components/AssertsForm.tsx
-import React, { useState } from 'react';
-import Autocomplete from '@mui/material/Autocomplete';
-import Box from '@mui/material/Box';
-import Button from '@mui/material/Button';
-import Delete from '@mui/icons-material/Delete';
-import IconButton from '@mui/material/IconButton';
-import Stack from '@mui/material/Stack';
-import TextField from '@mui/material/TextField';
-import Typography from '@mui/material/Typography';
-import type { Assertion, AssertionType } from '../../../../../types';
-
-interface AssertsFormProps {
-  onAdd: (asserts: Assertion[]) => void;
-  initialValues: Assertion[];
-}
-
-const assertTypes: AssertionType[] = [
-  'equals',
-  'contains',
-  'icontains',
-  'contains-all',
-  'contains-any',
-  'starts-with',
-  'regex',
-  'is-json',
-  'contains-json',
-  'javascript',
-  'python',
-  'similar',
-  'llm-rubric',
-  'webhook',
-  'rouge-n',
-  'rouge-s',
-  'rouge-l',
-  'not-equals',
-  'not-contains',
-  'not-icontains',
-  'not-contains-all',
-  'not-contains-any',
-  'not-starts-with',
-  'not-regex',
-  'not-is-json',
-  'not-contains-json',
-  'not-javascript',
-  'not-python',
-  'not-similar',
-  'not-llm-rubric',
-  'not-webhook',
-  'not-rouge-n',
-  'not-rouge-s',
-  'not-rouge-l',
-];
-
-const AssertsForm: React.FC<AssertsFormProps> = ({ onAdd, initialValues }) => {
-  const [asserts, setAsserts] = useState<Assertion[]>(initialValues || []);
-
-  const handleAdd = () => {
-    const newAsserts = [...asserts, { type: 'equals' as AssertionType, value: '' }];
-    setAsserts(newAsserts);
-    onAdd(newAsserts);
-  };
-
-  const handleRemoveAssert = (indexToRemove: number) => {
-    const newAsserts = asserts.filter((_, index) => index !== indexToRemove);
-    setAsserts(newAsserts);
-    onAdd(newAsserts);
-  };
-
-  return (
-    <>
-      <Typography variant="h6">Asserts</Typography>
-      <Box my={asserts.length > 0 ? 2 : 0}>
-        <Stack direction="column" spacing={2}>
-          {asserts.map((assert, index) => (
-            <Stack key={index} direction="row" spacing={2} alignItems="center">
-              <Autocomplete
-                value={assert.type}
-                options={assertTypes}
-                sx={{ minWidth: 200 }}
-                onChange={(event, newValue) => {
-                  const newType = newValue;
-                  const newAsserts = asserts.map((a, i) =>
-                    i === index ? { ...a, type: newType as AssertionType } : a,
-                  );
-                  setAsserts(newAsserts);
-                  onAdd(newAsserts);
-                }}
-                renderInput={(params) => <TextField {...params} label="Type" />}
-              />
-              <TextField
-                label="Value"
-                value={assert.value}
-                fullWidth
-                onChange={(e) => {
-                  const newValue = e.target.value;
-                  const newAsserts = asserts.map((a, i) =>
-                    i === index ? { ...a, value: newValue } : a,
-                  );
-                  setAsserts(newAsserts);
-                  onAdd(newAsserts);
-                }}
-              />
-              <IconButton onClick={() => handleRemoveAssert(index)} size="small">
-                <Delete />
-              </IconButton>
-            </Stack>
-          ))}
-        </Stack>
-      </Box>
-      <Button color="primary" onClick={handleAdd}>
-        Add Assert
-      </Button>
-    </>
-  );
-};
-
-export default AssertsForm;