npm - promptfoo - Versions diffs - 0.18.4 → 0.19.0 - Mend

promptfoo 0.18.4 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (156) hide show

package/src/web/{client/src/index.css → nextui/src/app/globals.css} RENAMED Viewed

@@ -1,6 +1,10 @@
-:root {
-  font-family: system-ui, Avenir, Helvetica, Arial, sans-serif;
+/* This CSS is common to all pages */
+* {
+  box-sizing: border-box;
+}
+:root {
   font-synthesis: none;
   text-rendering: optimizeLegibility;
   -webkit-font-smoothing: antialiased;
@@ -18,6 +22,8 @@
   --success-background-color: #d1ffd7;
   --variable-background-color: #f7f7f7;
   --header-background-color: #fffdf7;
+  --insert-highlight-color: #d4fcbc;
+  --delete-highlight-color: #fbb6c2;
 }
 /* Dark mode colors */
@@ -33,8 +39,20 @@
   --success-background-color: #216d2b;
   --variable-background-color: #333;
   --header-background-color: #333;
+  --insert-highlight-color: #4f8a34;
+  --delete-highlight-color: #8a3434;
 }
 html {
-  font-size: calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)));
+  font-size: 16px;
+  background-color: var(--background-color);
+  color: var(--text-color);
+}
+body {
+  margin: 0;
+}
+* {
+  box-sizing: border-box;
 }

package/src/web/nextui/src/app/layout.tsx ADDED Viewed

@@ -0,0 +1,25 @@
+import './globals.css';
+import type { Metadata } from 'next';
+import { Roboto } from 'next/font/google';
+import { PageShell } from './components/PageShell';
+const roboto = Roboto({
+  weight: ['400', '500', '700'],
+  style: ['normal'],
+  subsets: ['latin'],
+});
+export const metadata: Metadata = {
+  title: 'promptfoo',
+  description: 'LLM testing and evaluation',
+};
+export default function RootLayout({ children }: { children: React.ReactNode }) {
+  return (
+    <html lang="en">
+      <body className={roboto.className}>
+        <PageShell>{children}</PageShell>
+      </body>
+    </html>
+  );
+}

package/src/web/nextui/src/app/page.tsx ADDED Viewed

@@ -0,0 +1,7 @@
+import { redirect } from 'next/navigation';
+import './Home.css';
+export default function Page() {
+  redirect('/eval');
+}

package/src/web/nextui/src/app/setup/AssertsForm.tsx ADDED Viewed

@@ -0,0 +1,118 @@
+// src/components/AssertsForm.tsx
+import React, { useState } from 'react';
+import Autocomplete from '@mui/material/Autocomplete';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Delete from '@mui/icons-material/Delete';
+import IconButton from '@mui/material/IconButton';
+import Stack from '@mui/material/Stack';
+import TextField from '@mui/material/TextField';
+import Typography from '@mui/material/Typography';
+import type { Assertion, AssertionType } from '../../../../../types';
+interface AssertsFormProps {
+  onAdd: (asserts: Assertion[]) => void;
+  initialValues: Assertion[];
+}
+const assertTypes: AssertionType[] = [
+  'equals',
+  'contains',
+  'icontains',
+  'contains-all',
+  'contains-any',
+  'starts-with',
+  'regex',
+  'is-json',
+  'contains-json',
+  'javascript',
+  'python',
+  'similar',
+  'llm-rubric',
+  'webhook',
+  'rouge-n',
+  'rouge-s',
+  'rouge-l',
+  'not-equals',
+  'not-contains',
+  'not-icontains',
+  'not-contains-all',
+  'not-contains-any',
+  'not-starts-with',
+  'not-regex',
+  'not-is-json',
+  'not-contains-json',
+  'not-javascript',
+  'not-python',
+  'not-similar',
+  'not-llm-rubric',
+  'not-webhook',
+  'not-rouge-n',
+  'not-rouge-s',
+  'not-rouge-l',
+];
+const AssertsForm: React.FC<AssertsFormProps> = ({ onAdd, initialValues }) => {
+  const [asserts, setAsserts] = useState<Assertion[]>(initialValues || []);
+  const handleAdd = () => {
+    const newAsserts = [...asserts, { type: 'equals' as AssertionType, value: '' }];
+    setAsserts(newAsserts);
+    onAdd(newAsserts);
+  };
+  const handleRemoveAssert = (indexToRemove: number) => {
+    const newAsserts = asserts.filter((_, index) => index !== indexToRemove);
+    setAsserts(newAsserts);
+    onAdd(newAsserts);
+  };
+  return (
+    <>
+      <Typography variant="h6">Asserts</Typography>
+      <Box my={asserts.length > 0 ? 2 : 0}>
+        <Stack direction="column" spacing={2}>
+          {asserts.map((assert, index) => (
+            <Stack key={index} direction="row" spacing={2} alignItems="center">
+              <Autocomplete
+                value={assert.type}
+                options={assertTypes}
+                sx={{ minWidth: 200 }}
+                onChange={(event, newValue) => {
+                  const newType = newValue;
+                  const newAsserts = asserts.map((a, i) =>
+                    i === index ? { ...a, type: newType as AssertionType } : a,
+                  );
+                  setAsserts(newAsserts);
+                  onAdd(newAsserts);
+                }}
+                renderInput={(params) => <TextField {...params} label="Type" />}
+              />
+              <TextField
+                label="Value"
+                value={assert.value}
+                fullWidth
+                onChange={(e) => {
+                  const newValue = e.target.value;
+                  const newAsserts = asserts.map((a, i) =>
+                    i === index ? { ...a, value: newValue } : a,
+                  );
+                  setAsserts(newAsserts);
+                  onAdd(newAsserts);
+                }}
+              />
+              <IconButton onClick={() => handleRemoveAssert(index)} size="small">
+                <Delete />
+              </IconButton>
+            </Stack>
+          ))}
+        </Stack>
+      </Box>
+      <Button color="primary" onClick={handleAdd}>
+        Add Assert
+      </Button>
+    </>
+  );
+};
+export default AssertsForm;

package/src/web/nextui/src/app/setup/PromptDialog.tsx ADDED Viewed

@@ -0,0 +1,77 @@
+import React from 'react';
+import {
+  Dialog,
+  DialogTitle,
+  DialogContent,
+  DialogActions,
+  TextField,
+  Button,
+} from '@mui/material';
+interface PromptDialogProps {
+  open: boolean;
+  prompt: string;
+  index: number;
+  onAdd: (prompt: string) => void;
+  onCancel: () => void;
+}
+const PromptDialog: React.FC<PromptDialogProps> = ({ open, prompt, index, onAdd, onCancel }) => {
+  const [editingPrompt, setEditingPrompt] = React.useState(prompt);
+  const textFieldRef = React.useRef<HTMLInputElement>(null);
+  React.useEffect(() => {
+    setEditingPrompt(prompt);
+  }, [prompt]);
+  const handleAdd = (close: boolean) => {
+    onAdd(editingPrompt);
+    setEditingPrompt('');
+    if (close) {
+      onCancel();
+    } else if (textFieldRef.current) {
+      textFieldRef.current.focus();
+    }
+  };
+  return (
+    <Dialog open={open} onClose={onCancel} fullWidth maxWidth="md">
+      <DialogTitle>{`Edit Prompt ${index + 1}`}</DialogTitle>
+      <DialogContent>
+        <TextField
+          value={editingPrompt}
+          onChange={(e) => setEditingPrompt(e.target.value)}
+          fullWidth
+          margin="normal"
+          multiline
+          placeholder="The quick brown {{animal1}} jumps over the lazy {{animal2}}."
+          helperText="Tip: use the {{varname}} syntax to add variables to your prompt."
+          inputRef={textFieldRef}
+        />
+      </DialogContent>
+      <DialogActions>
+        <Button
+          onClick={handleAdd.bind(null, true)}
+          color="primary"
+          variant="contained"
+          disabled={!editingPrompt.length}
+        >
+          Add
+        </Button>
+        <Button
+          onClick={handleAdd.bind(null, false)}
+          color="primary"
+          variant="contained"
+          disabled={!editingPrompt.length}
+        >
+          Add Another
+        </Button>
+        <Button onClick={onCancel} color="secondary">
+          Cancel
+        </Button>
+      </DialogActions>
+    </Dialog>
+  );
+};
+export default PromptDialog;

package/src/web/nextui/src/app/setup/PromptsSection.tsx ADDED Viewed

@@ -0,0 +1,190 @@
+import React, { useState, useRef, useEffect } from 'react';
+import Button from '@mui/material/Button';
+import Typography from '@mui/material/Typography';
+import IconButton from '@mui/material/IconButton';
+import Table from '@mui/material/Table';
+import TableBody from '@mui/material/TableBody';
+import TableCell from '@mui/material/TableCell';
+import TableContainer from '@mui/material/TableContainer';
+import TableRow from '@mui/material/TableRow';
+import Tooltip from '@mui/material/Tooltip';
+import Stack from '@mui/material/Stack';
+import Edit from '@mui/icons-material/Edit';
+import Delete from '@mui/icons-material/Delete';
+import Publish from '@mui/icons-material/Publish';
+import Copy from '@mui/icons-material/ContentCopy';
+import PromptDialog from './PromptDialog';
+import { useStore } from '../../util/store';
+const PromptsSection: React.FC = () => {
+  const [promptDialogOpen, setPromptDialogOpen] = useState(false);
+  const [editingPromptIndex, setEditingPromptIndex] = useState<number | null>(null);
+  const { prompts, setPrompts } = useStore();
+  const newPromptInputRef = useRef<HTMLInputElement>(null);
+  useEffect(() => {
+    if (editingPromptIndex !== null && editingPromptIndex > 0 && newPromptInputRef.current) {
+      newPromptInputRef.current.focus();
+    }
+  }, [editingPromptIndex]);
+  const handleEditPrompt = (index: number) => {
+    setEditingPromptIndex(index);
+    setPromptDialogOpen(true);
+  };
+  const handleAddPromptFromFile = (event: React.ChangeEvent<HTMLInputElement>) => {
+    event.stopPropagation();
+    event.preventDefault();
+    const file = event.target.files?.[0];
+    if (file) {
+      const reader = new FileReader();
+      reader.onload = (e) => {
+        const text = e.target?.result?.toString();
+        if (text) {
+          setPrompts([...prompts, text]);
+        }
+      };
+      reader.readAsText(file);
+    }
+  };
+  const handleDuplicatePrompt = (event: React.MouseEvent, index: number) => {
+    event.stopPropagation();
+    const duplicatedPrompt = prompts[index];
+    setPrompts([...prompts, duplicatedPrompt]);
+  };
+  const handleChangePrompt = (index: number, newPrompt: string) => {
+    setPrompts(prompts.map((p, i) => (i === index ? newPrompt : p)));
+  };
+  const handleRemovePrompt = (event: React.MouseEvent, indexToRemove: number) => {
+    event.stopPropagation();
+    if (confirm('Are you sure you want to remove this prompt?')) {
+      setPrompts(prompts.filter((_, index) => index !== indexToRemove));
+    }
+  };
+  return (
+    <div>
+      <Stack direction="row" spacing={2} justifyContent="space-between">
+        <Typography variant="h5">Prompts</Typography>
+        <div>
+          <label htmlFor={`file-input-add-prompt`}>
+            <Tooltip title="Upload prompt from file">
+              <span>
+                <IconButton component="span">
+                  <Publish />
+                </IconButton>
+                <input
+                  id={`file-input-add-prompt`}
+                  type="file"
+                  accept=".txt,.md"
+                  onChange={handleAddPromptFromFile}
+                  style={{ display: 'none' }}
+                />
+              </span>
+            </Tooltip>
+          </label>
+          <Button
+            color="primary"
+            onClick={() => {
+              setPromptDialogOpen(true);
+            }}
+            variant="contained"
+          >
+            Add Prompt
+          </Button>
+        </div>
+      </Stack>
+      <TableContainer>
+        <Table>
+          <TableBody>
+            {prompts.length === 0 ? (
+              <TableRow>
+                <TableCell colSpan={2} align="center">
+                  No prompts added yet.
+                </TableCell>
+              </TableRow>
+            ) : (
+              prompts.map((prompt, index) => (
+                <TableRow
+                  key={index}
+                  sx={{
+                    '&:hover': {
+                      backgroundColor: 'rgba(0, 0, 0, 0.04)',
+                      cursor: 'pointer',
+                    },
+                  }}
+                  onClick={() => handleEditPrompt(index)}
+                >
+                  <TableCell>
+                    <Typography variant="body2">
+                      {`Prompt #${index + 1}: `}
+                      {(prompt.length > 250 ? prompt.slice(0, 250) + ' ...' : prompt)
+                        .split(/({{\w+}})/g)
+                        .map((part, i) =>
+                          /{{\w+}}/g.test(part) ? (
+                            <span
+                              key={i}
+                              style={{
+                                backgroundColor: 'linen',
+                                padding: '0.25rem',
+                                borderRadius: '4px',
+                              }}
+                            >
+                              {part}
+                            </span>
+                          ) : (
+                            part
+                          ),
+                        )}
+                    </Typography>
+                  </TableCell>
+                  <TableCell align="right" sx={{ minWidth: 150 }}>
+                    <IconButton onClick={() => handleEditPrompt(index)} size="small">
+                      <Edit />
+                    </IconButton>
+                    <IconButton
+                      onClick={(event) => handleDuplicatePrompt(event, index)}
+                      size="small"
+                    >
+                      <Copy />
+                    </IconButton>
+                    <IconButton onClick={(event) => handleRemovePrompt(event, index)} size="small">
+                      <Delete />
+                    </IconButton>
+                  </TableCell>
+                </TableRow>
+              ))
+            )}
+          </TableBody>
+        </Table>
+      </TableContainer>
+      <PromptDialog
+        open={promptDialogOpen}
+        prompt={editingPromptIndex !== null ? prompts[editingPromptIndex] : ''}
+        index={editingPromptIndex !== null ? editingPromptIndex : 0}
+        onAdd={(newPrompt) => {
+          if (editingPromptIndex !== null) {
+            handleChangePrompt(editingPromptIndex, newPrompt);
+          } else {
+            setPrompts([...prompts, newPrompt]);
+          }
+          setEditingPromptIndex(null);
+        }}
+        onCancel={() => {
+          setEditingPromptIndex(null);
+          setPromptDialogOpen(false);
+        }}
+      />
+    </div>
+  );
+};
+export default PromptsSection;

package/src/web/nextui/src/app/setup/ProviderConfigDialog.tsx ADDED Viewed

@@ -0,0 +1,99 @@
+import React from 'react';
+import {
+  Box,
+  Dialog,
+  DialogTitle,
+  DialogContent,
+  TextField,
+  DialogActions,
+  Button,
+} from '@mui/material';
+import { ProviderConfig } from '../../../../../types';
+interface ProviderConfigDialogProps {
+  open: boolean;
+  providerId: string;
+  config: ProviderConfig['config'];
+  onClose: () => void;
+  onSave: (config: ProviderConfig['config']) => void;
+}
+const ProviderConfigDialog: React.FC<ProviderConfigDialogProps> = ({
+  open,
+  providerId,
+  config,
+  onClose,
+  onSave,
+}) => {
+  const [localConfig, setLocalConfig] = React.useState(config);
+  React.useEffect(() => {
+    setLocalConfig(config);
+  }, [config]);
+  const handleSave = () => {
+    onSave(localConfig);
+  };
+  return (
+    <Dialog open={open} onClose={onClose}>
+      <DialogTitle>Edit {providerId}</DialogTitle>
+      <DialogContent>
+        {Object.keys(localConfig).map((key) => {
+          const value = localConfig[key];
+          let handleChange;
+          if (
+            typeof value === 'number' ||
+            typeof value === 'boolean' ||
+            typeof value === 'string'
+          ) {
+            if (typeof value === 'number') {
+              handleChange = (e: React.ChangeEvent<HTMLInputElement>) =>
+                setLocalConfig({ ...localConfig, [key]: parseFloat(e.target.value) });
+            } else if (typeof value === 'boolean') {
+              handleChange = (e: React.ChangeEvent<HTMLInputElement>) =>
+                setLocalConfig({ ...localConfig, [key]: e.target.value === 'true' });
+            } else {
+              handleChange = (e: React.ChangeEvent<HTMLInputElement>) =>
+                setLocalConfig({ ...localConfig, [key]: e.target.value });
+            }
+            return (
+              <Box key={key} my={2}>
+                <TextField
+                  label={key}
+                  value={value}
+                  onChange={handleChange}
+                  fullWidth
+                  type={typeof value === 'number' ? 'number' : 'text'}
+                />
+              </Box>
+            );
+          } else {
+            return (
+              <Box key={key} my={2}>
+                <TextField
+                  label={key}
+                  value={JSON.stringify(value)}
+                  onChange={(e: React.ChangeEvent<HTMLInputElement>) =>
+                    setLocalConfig({ ...localConfig, [key]: JSON.parse(e.target.value) })
+                  }
+                  fullWidth
+                  multiline
+                  minRows={3}
+                />
+              </Box>
+            );
+          }
+        })}
+      </DialogContent>
+      <DialogActions>
+        <Button onClick={onClose}>Cancel</Button>
+        <Button onClick={handleSave}>Save</Button>
+      </DialogActions>
+    </Dialog>
+  );
+};
+export default ProviderConfigDialog;