promptfoo 0.18.4 → 0.19.0

package/src/web/nextui/package.json

@@ -0,0 +1,45 @@
+{
+  "name": "nextui",
+  "version": "0.1.0",
+  "private": true,
+  "scripts": {
+    "dev": "next dev",
+    "build": "next build",
+    "start": "next start",
+    "lint": "next lint"
+  },
+  "dependencies": {
+    "@emotion/react": "^11.11.1",
+    "@emotion/styled": "^11.11.0",
+    "@mui/icons-material": "^5.14.3",
+    "@mui/material": "^5.14.4",
+    "@tanstack/react-table": "^8.9.3",
+    "@types/diff": "^5.0.3",
+    "@types/node": "20.4.10",
+    "@types/react": "18.2.20",
+    "@types/react-dom": "18.2.7",
+    "@types/react-syntax-highlighter": "^15.5.7",
+    "@types/uuid": "^9.0.2",
+    "debounce": "^1.2.1",
+    "diff": "^5.1.0",
+    "eslint": "8.47.0",
+    "eslint-config-next": "13.4.13",
+    "js-yaml": "^4.1.0",
+    "next": "13.4.13",
+    "opener": "^1.5.2",
+    "react": "18.2.0",
+    "react-dnd": "^16.0.1",
+    "react-dnd-html5-backend": "^16.0.1",
+    "react-dom": "18.2.0",
+    "react-syntax-highlighter": "^15.5.0",
+    "socket.io": "^4.7.2",
+    "socket.io-client": "^4.7.2",
+    "tiny-invariant": "^1.3.1",
+    "typescript": "5.1.6",
+    "uuid": "^9.0.0",
+    "zustand": "^4.4.1"
+  },
+  "devDependencies": {
+    "@types/js-yaml": "^4.0.5"
+  }
+}

package/src/web/nextui/src/app/Home.css

@@ -0,0 +1,3 @@
+.container {
+  margin: 2rem;
+}

package/src/web/nextui/src/app/api/route.ts

@@ -0,0 +1,6 @@
+import { NextResponse } from 'next/server';
+
+export async function GET() {
+  const data = { message: Math.random() < 0.5 ? 'Hello World!' : 'Hello mom' };
+  return NextResponse.json({ data });
+}

package/src/web/{client/src/NavBar.css → nextui/src/app/components/DarkMode.css}

@@ -7,6 +7,7 @@ nav {
 }
 
 .dark-mode-toggle {
+  display: flex;
   background-color: transparent;
   border: none;
   color: var(--text-color);

package/src/web/{client/src/NavBar.tsx → nextui/src/app/components/DarkMode.tsx}

@@ -1,9 +1,7 @@
-import Logo from './Logo';
-
 import DarkModeIcon from '@mui/icons-material/DarkMode';
 import LightModeIcon from '@mui/icons-material/LightMode';
 
-import './NavBar.css';
+import './DarkMode.css';
 
 interface NavbarProps {
   darkMode: boolean;
@@ -12,11 +10,8 @@ interface NavbarProps {
 
 export default function NavBar({ darkMode, onToggleDarkMode }: NavbarProps) {
   return (
-    <nav>
-      <Logo />
-      <div className="dark-mode-toggle" onClick={onToggleDarkMode}>
-        {darkMode ? <DarkModeIcon /> : <LightModeIcon />}
-      </div>
-    </nav>
+    <div className="dark-mode-toggle" onClick={onToggleDarkMode}>
+      {darkMode ? <DarkModeIcon /> : <LightModeIcon />}
+    </div>
   );
 }

package/src/web/nextui/src/app/components/Logo.css

@@ -0,0 +1,32 @@
+.logo {
+  display: flex;
+  font-family: mono;
+  font-size: 0.8rem;
+  align-items: center;
+  gap: 8px;
+  background-color: #f0f0f0;
+  padding: 0 2rem 0 0;
+  border-radius: 0.5rem;
+}
+
+[data-theme='dark'] .logo {
+  background-color: #333;
+  border-color: #444;
+}
+
+.logo img {
+  width: 25px;
+  margin-top: 2px;
+}
+
+[data-theme='dark'] .logo img {
+  filter: invert(1);
+}
+
+.logo span {
+  color: var(--text-color);
+}
+
+[data-theme='dark'] .logo span {
+  color: #f0f0f0;
+}

package/src/web/nextui/src/app/components/PageShell.css

@@ -0,0 +1,33 @@
+.nav {
+  padding: 0.25rem 0 0.25rem 1rem;
+  background-color: #eee;
+  margin-bottom: 1rem;
+}
+
+[data-theme='dark'] .nav {
+  background-color: #333;
+}
+
+.nav a {
+  padding: 0 0.5rem;
+  text-decoration: none;
+  color: #000;
+  align-self: center;
+}
+
+[data-theme='dark'] .nav a {
+  color: #f0f0f0;
+}
+
+.nav div:last-child {
+  margin-left: auto;
+  margin-right: 0.5rem;
+}
+
+.nav a:hover {
+  text-decoration: underline;
+}
+
+[data-theme='dark'] .nav a:hover {
+  color: #ddd;
+}

package/src/web/nextui/src/app/components/PageShell.tsx

@@ -0,0 +1,87 @@
+'use client';
+
+import React from 'react';
+import Link from 'next/link';
+import useMediaQuery from '@mui/material/useMediaQuery';
+import { Stack } from '@mui/material';
+import { ThemeProvider, createTheme } from '@mui/material/styles';
+
+import Logo from './Logo';
+import DarkMode from './DarkMode';
+
+import './PageShell.css';
+
+export { PageShell };
+
+function PageShell({ children }: { children: React.ReactNode }) {
+  const prefersDarkMode = useMediaQuery('(prefers-color-scheme: dark)');
+  const [darkMode, setDarkMode] = React.useState(prefersDarkMode);
+
+  const theme = React.useMemo(
+    () =>
+      createTheme({
+        typography: {
+          fontFamily: 'inherit',
+        },
+        palette: {
+          mode: darkMode ? 'dark' : 'light',
+        },
+      }),
+    [darkMode],
+  );
+
+  const toggleDarkMode = () => {
+    setDarkMode(!darkMode);
+    if (!darkMode) {
+      document.documentElement.setAttribute('data-theme', 'dark');
+    } else {
+      document.documentElement.removeAttribute('data-theme');
+    }
+  };
+
+  React.useEffect(() => {
+    if (prefersDarkMode) {
+      document.documentElement.setAttribute('data-theme', 'dark');
+    }
+  }, [prefersDarkMode]);
+
+  return (
+    <React.StrictMode>
+      <ThemeProvider theme={theme}>
+        <Layout>
+          <Navigation darkMode={darkMode} onToggleDarkMode={toggleDarkMode} />
+          <div>{children}</div>
+        </Layout>
+      </ThemeProvider>
+    </React.StrictMode>
+  );
+}
+
+function Layout({ children }: { children: React.ReactNode }) {
+  return <div>{children}</div>;
+}
+
+function Navigation({
+  darkMode,
+  onToggleDarkMode,
+}: {
+  darkMode: boolean;
+  onToggleDarkMode: () => void;
+}) {
+  if (process.env.NEXT_PUBLIC_NO_BROWSING) {
+    return (
+      <Stack direction="row" spacing={2} className="nav">
+        <Logo />
+        <DarkMode darkMode={darkMode} onToggleDarkMode={onToggleDarkMode} />
+      </Stack>
+    );
+  }
+  return (
+    <Stack direction="row" spacing={2} className="nav">
+      <Logo />
+      <Link href="/setup">New Eval</Link>
+      <Link href="/eval">View Evals</Link>
+      <DarkMode darkMode={darkMode} onToggleDarkMode={onToggleDarkMode} />
+    </Stack>
+  );
+}

package/src/web/{client/src → nextui/src/app/eval}/ConfigModal.tsx

@@ -1,13 +1,16 @@
 import React from 'react';
+import Box from '@mui/material/Box';
+import Button from '@mui/material/Button';
+import Check from '@mui/icons-material/Check';
 import Dialog from '@mui/material/Dialog';
-import DialogTitle from '@mui/material/DialogTitle';
-import DialogContent from '@mui/material/DialogContent';
 import DialogActions from '@mui/material/DialogActions';
-import Button from '@mui/material/Button';
+import DialogContent from '@mui/material/DialogContent';
+import DialogTitle from '@mui/material/DialogTitle';
+import FileCopy from '@mui/icons-material/FileCopy';
+import IconButton from '@mui/material/IconButton';
 import Typography from '@mui/material/Typography';
+
 import { useStore } from './store';
-import { IconButton, Box } from '@mui/material';
-import { FileCopy, Check } from '@mui/icons-material';
 
 interface ConfigModalProps {
   open: boolean;

package/src/web/nextui/src/app/eval/Eval.css

@@ -0,0 +1,13 @@
+body {
+  background-color: var(--background-color);
+  color: var(--text-color);
+}
+
+.loading {
+  display: flex;
+  flex-direction: column;
+  gap: 1.5rem;
+  justify-content: center;
+  align-items: center;
+  height: 9rem;
+}

package/src/web/nextui/src/app/eval/Eval.tsx

@@ -0,0 +1,79 @@
+'use client';
+
+import * as React from 'react';
+import CircularProgress from '@mui/material/CircularProgress';
+import { io as SocketIOClient } from 'socket.io-client';
+
+import ResultsView from './ResultsView';
+import { API_BASE_URL } from '@/util/api';
+import { useStore } from './store';
+
+import type { EvalTable, SharedResults } from './types';
+
+import './Eval.css';
+
+interface EvalOptions {
+  preloadedData?: SharedResults;
+}
+
+export default function Eval({ preloadedData }: EvalOptions) {
+  const { table, setTable, setConfig } = useStore();
+  const [loaded, setLoaded] = React.useState<boolean>(false);
+  const [recentFiles, setRecentFiles] = React.useState<string[]>([]);
+
+  const fetchRecentFiles = async () => {
+    if (!window.location.href.includes('localhost')) {
+      return;
+    }
+    const resp = await fetch(`${API_BASE_URL}/results`);
+    const body = await resp.json();
+    setRecentFiles(body.data);
+  };
+
+  const handleRecentFileSelection = async (file: string) => {
+    const resp = await fetch(`${API_BASE_URL}/results/${file}`);
+    const body = await resp.json();
+    setTable(body.data.results.table);
+    setConfig(body.data.config);
+  };
+
+  React.useEffect(() => {
+    const socket = SocketIOClient(API_BASE_URL);
+
+    if (preloadedData) {
+      setTable(preloadedData.data.results?.table as EvalTable);
+      setConfig(preloadedData.data.config);
+      setLoaded(true);
+    } else {
+      socket.on('init', (data) => {
+        console.log('Initialized socket connection', data);
+        setLoaded(true);
+        setTable(data.results.table);
+        setConfig(data.config);
+        fetchRecentFiles();
+      });
+
+      socket.on('update', (data) => {
+        console.log('Received data update', data);
+        setTable(data.results.table);
+        setConfig(data.config);
+        fetchRecentFiles();
+      });
+    }
+
+    return () => {
+      socket.disconnect();
+    };
+  }, [setTable, setConfig, preloadedData]);
+
+  return loaded && table ? (
+    <ResultsView recentFiles={recentFiles} onRecentFileSelected={handleRecentFileSelection} />
+  ) : (
+    <div className="loading">
+      <div>
+        <CircularProgress size={22} />
+      </div>
+      <div>Loading eval data</div>
+    </div>
+  );
+}

package/src/web/{client/src → nextui/src/app/eval}/EvalOutputPromptDialog.tsx

@@ -5,7 +5,7 @@ import Dialog from '@mui/material/Dialog';
 import DialogActions from '@mui/material/DialogActions';
 import DialogContent from '@mui/material/DialogContent';
 import DialogTitle from '@mui/material/DialogTitle';
-import TextareaAutosize from '@mui/base/TextareaAutosize';
+import TextareaAutosize from '@mui/material/TextareaAutosize';
 import IconButton from '@mui/material/IconButton';
 import ContentCopyIcon from '@mui/icons-material/ContentCopy';
 import CheckIcon from '@mui/icons-material/Check';
@@ -17,7 +17,7 @@ import TableHead from '@mui/material/TableHead';
 import TableRow from '@mui/material/TableRow';
 import Typography from '@mui/material/Typography';
 
-import type { GradingResult } from '../../../types';
+import type { GradingResult } from './types';
 
 interface EvalOutputPromptDialogProps {
   open: boolean;

package/src/web/{client/src → nextui/src/app/eval}/ResultsTable.css

@@ -1,15 +1,3 @@
-* {
-  box-sizing: border-box;
-}
-
-html {
-  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif,
-    'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
-  font-size: 16px;
-  background-color: var(--background-color);
-  color: var(--text-color);
-}
-
 table,
 .divTable {
   border: 1px solid var(--table-border-color);
@@ -20,6 +8,16 @@ table,
   box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
 }
 
+ins {
+  background-color: var(--insert-highlight-color);
+  text-decoration: none;
+}
+
+del {
+  background-color: var(--delete-highlight-color);
+  text-decoration: strikethrough;
+}
+
 .tr {
   display: flex;
 }

package/src/web/{client/src → nextui/src/app/eval}/ResultsTable.tsx

@@ -1,4 +1,5 @@
 import * as React from 'react';
+import { diffSentences, diffJson, diffWords } from 'diff';
 
 import './index.css';
 
@@ -12,17 +13,17 @@ import {
 import Checkbox from '@mui/material/Checkbox';
 import FormControlLabel from '@mui/material/FormControlLabel';
 
-import { useStore } from './store.js';
+import { useStore } from './store';
 
 import type { CellContext, VisibilityState } from '@tanstack/table-core';
 
 import EvalOutputPromptDialog from './EvalOutputPromptDialog';
 
-import type { EvalRow, EvalRowOutput, FilterMode } from './types.js';
+import type { EvalRow, EvaluateTableOutput, FilterMode, GradingResult } from './types';
 
 import './ResultsTable.css';
 
-function formatRowOutput(output: EvalRowOutput | string) {
+function formatRowOutput(output: EvaluateTableOutput | string) {
   if (typeof output === 'string') {
     // Backwards compatibility for 0.15.0 breaking change. Remove eventually.
     const pass = output.startsWith('[PASS]');
@@ -86,7 +87,7 @@ function TruncatedText({ text: rawText, maxLength }: TruncatedTextProps) {
 }
 
 interface PromptOutputProps {
-  output: EvalRowOutput;
+  output: EvaluateTableOutput;
   maxTextLength: number;
   rowIndex: number;
   promptIndex: number;
@@ -99,7 +100,9 @@ function EvalOutputCell({
   rowIndex,
   promptIndex,
   onRating,
-}: PromptOutputProps) {
+  firstOutput,
+  filterMode,
+}: PromptOutputProps & { firstOutput: EvaluateTableOutput; filterMode: FilterMode }) {
   const [openPrompt, setOpen] = React.useState(false);
   const handlePromptOpen = () => {
     setOpen(true);
@@ -115,6 +118,42 @@ function EvalOutputCell({
     text = chunks.slice(1).join('---');
   }
 
+  if (filterMode === 'different' && firstOutput) {
+    let firstOutputText =
+      typeof firstOutput.text === 'string' ? firstOutput.text : JSON.stringify(firstOutput.text);
+
+    if (firstOutputText.includes('---')) {
+      firstOutputText = firstOutputText.split('---').slice(1).join('---');
+    }
+
+    let diffResult;
+    try {
+      // Try parsing the texts as JSON
+      JSON.parse(firstOutputText);
+      JSON.parse(text);
+      // If no errors are thrown, the texts are valid JSON
+      diffResult = diffJson(firstOutputText, text);
+    } catch (error) {
+      // If an error is thrown, the texts are not valid JSON
+      if (firstOutputText.includes('. ') && text.includes('. ')) {
+        // If the texts contain a period, they are considered as prose
+        diffResult = diffSentences(firstOutputText, text);
+      } else {
+        // If the texts do not contain a period, use diffWords
+        diffResult = diffWords(firstOutputText, text);
+      }
+    }
+    text = diffResult
+      .map((part: { added?: boolean; removed?: boolean; value: string }) =>
+        part.added
+          ? `<ins>${part.value}</ins>`
+          : part.removed
+          ? `<del>${part.value}</del>`
+          : part.value,
+      )
+      .join('');
+  }
+
   const handleClick = (isPass: boolean) => {
     onRating(rowIndex, promptIndex, isPass);
   };
@@ -239,7 +278,9 @@ export default function ResultsTable({
   const numGoodAsserts = head.prompts.map((_, idx) =>
     body.reduce((acc, row) => {
       const componentResults = row.outputs[idx].gradingResult?.componentResults;
-      return acc + (componentResults ? componentResults.filter((r) => r.pass).length : 0);
+      return (
+        acc + (componentResults ? componentResults.filter((r: GradingResult) => r.pass).length : 0)
+      );
     }, 0),
   );
 
@@ -341,13 +382,15 @@ export default function ResultsTable({
               </>
             );
           },
-          cell: (info: CellContext<EvalRow, EvalRowOutput>) => (
+          cell: (info: CellContext<EvalRow, EvaluateTableOutput>) => (
             <EvalOutputCell
-              output={info.getValue() as unknown as EvalRowOutput}
+              output={info.getValue() as unknown as EvaluateTableOutput}
               maxTextLength={maxTextLength}
               rowIndex={info.row.index}
               promptIndex={idx}
               onRating={handleRating}
+              firstOutput={filteredBody[info.row.index].outputs[0]}
+              filterMode={filterMode}
             />
           ),
         }),
@@ -394,13 +437,13 @@ export default function ResultsTable({
       }}
     >
       <thead>
-        {reactTable.getHeaderGroups().map((headerGroup) => (
+        {reactTable.getHeaderGroups().map((headerGroup: any) => (
           <tr key={headerGroup.id} className="header">
-            {headerGroup.headers.map((header) => {
+            {headerGroup.headers.map((header: any) => {
               return (
                 <th
+                  key={header.id}
                   {...{
-                    key: header.id,
                     colSpan: header.colSpan,
                     style: {
                       width: header.getSize(),
@@ -424,11 +467,11 @@ export default function ResultsTable({
         ))}
       </thead>
       <tbody>
-        {reactTable.getRowModel().rows.map((row, rowIndex) => {
+        {reactTable.getRowModel().rows.map((row: any, rowIndex: any) => {
           let colBorderDrawn = false;
           return (
             <tr key={row.id}>
-              {row.getVisibleCells().map((cell) => {
+              {row.getVisibleCells().map((cell: any) => {
                 const isVariableCol = cell.column.id.startsWith('Variable');
                 const shouldDrawColBorder = !isVariableCol && !colBorderDrawn;
                 if (shouldDrawColBorder) {
@@ -437,8 +480,8 @@ export default function ResultsTable({
                 const shouldDrawRowBorder = rowIndex === 0 && !isVariableCol;
                 return (
                   <td
+                    key={cell.id}
                     {...{
-                      key: cell.id,
                       style: {
                         width: cell.column.getSize(),
                       },

package/src/web/{client/src → nextui/src/app/eval}/ResultsView.tsx

@@ -21,13 +21,13 @@ import ShareIcon from '@mui/icons-material/Share';
 import VisibilityIcon from '@mui/icons-material/Visibility';
 import { styled } from '@mui/system';
 
-import ResultsTable from './ResultsTable.js';
+import ResultsTable from './ResultsTable';
 import ConfigModal from './ConfigModal';
 import ShareModal from './ShareModal';
-import { useStore } from './store.js';
+import { useStore } from './store';
 
 import type { VisibilityState } from '@tanstack/table-core';
-import type { FilterMode } from './types.js';
+import type { FilterMode } from './types';
 
 const ResponsiveStack = styled(Stack)(({ theme }) => ({
   maxWidth: '100%',
@@ -171,7 +171,7 @@ export default function ResultsView({ recentFiles, onRecentFileSelected }: Resul
   }, [head]);
 
   return (
-    <div>
+    <div style={{ marginLeft: '1rem', marginRight: '1rem' }}>
       <Paper py="md">
         <ResponsiveStack direction="row" spacing={4} alignItems="center">
           <Box>

package/src/web/nextui/src/app/eval/[id]/not-found.tsx

@@ -0,0 +1,5 @@
+import './page.css';
+
+export default function NotFound() {
+  return <div className="error">Eval not found</div>;
+}

package/src/web/nextui/src/app/eval/[id]/page.css

@@ -0,0 +1,9 @@
+.error {
+  display: flex;
+  flex-direction: column;
+  gap: 1.5rem;
+  justify-content: center;
+  align-items: center;
+  height: 9rem;
+  font-size: 1.5rem;
+}

package/src/web/nextui/src/app/eval/[id]/page.tsx

@@ -0,0 +1,20 @@
+import React from 'react';
+import { notFound } from 'next/navigation';
+
+import Eval from '../Eval';
+
+import './page.css';
+
+export async function generateStaticParams() {
+  return [];
+}
+
+export default async function Page({ params }: { params: { id: string } }) {
+  const response = await fetch(`https://api.promptfoo.dev/eval/${params.id}`);
+  if (!response.ok) {
+    notFound();
+  }
+  const data = await response.json();
+
+  return <Eval preloadedData={data} />;
+}

package/src/web/nextui/src/app/eval/page.tsx

@@ -0,0 +1,8 @@
+import React from 'react';
+
+import Eval from './Eval';
+
+export default function Page() {
+  // Just return local eval from websocket
+  return <Eval />;
+}

package/src/web/{client/src → nextui/src/app/eval}/store.ts

@@ -1,6 +1,6 @@
-import create from 'zustand';
+import { create } from 'zustand';
 
-import type { EvalTable, UnifiedConfig } from './types.js';
+import type { EvalTable, UnifiedConfig } from './types';
 
 interface TableState {
   table: EvalTable | null;

package/src/web/nextui/src/app/eval/types.ts

@@ -0,0 +1,20 @@
+import type { Prompt, EvaluateTableOutput } from '../../../../../types';
+
+export type EvalHead = {
+  prompts: Prompt[];
+  vars: string[];
+};
+
+export type EvalRow = {
+  outputs: EvaluateTableOutput[];
+  vars: string[]; // model outputs
+};
+
+export type EvalTable = {
+  head: EvalHead;
+  body: EvalRow[];
+};
+
+export type FilterMode = 'all' | 'failures' | 'different';
+
+export * from '../../../../../types';