promptfoo 0.3.0 → 0.4.0

package/src/web/client/src/App.tsx

@@ -0,0 +1,43 @@
+import * as React from 'react';
+
+import { io as SocketIOClient } from 'socket.io-client';
+
+import ResultsView from './ResultsView.js';
+import NavBar from './NavBar.js';
+import { useStore } from './store.js';
+
+import './App.css';
+
+function App() {
+  const { table, setTable } = useStore();
+  const [loaded, setLoaded] = React.useState<boolean>(false);
+
+  React.useEffect(() => {
+    const socket = SocketIOClient(`http://${window.location.host}`);
+    //const socket = SocketIOClient(`http://localhost:15500`);
+
+    socket.on('init', (data) => {
+      console.log('Initialized socket connection');
+      setLoaded(true);
+      setTable(data.table);
+    });
+
+    socket.on('update', (data) => {
+      console.log('Received data update');
+      setTable(data.table);
+    });
+
+    return () => {
+      socket.disconnect();
+    };
+  }, [loaded, setTable]);
+
+  return (
+    <>
+      <NavBar />
+      {loaded && table ? <ResultsView /> : <div>Loading...</div>}
+    </>
+  );
+}
+
+export default App;

package/src/web/client/src/Logo.css

@@ -0,0 +1,13 @@
+.logo {
+  display: flex;
+  align-items: center;
+  gap: 4px;
+}
+
+.logo img {
+  width: 30px;
+}
+
+.logo span {
+  margin-bottom: 6px;
+}

package/src/web/client/src/Logo.tsx

@@ -0,0 +1,11 @@
+import Box from '@mui/material/Box';
+
+import './Logo.css';
+
+export default function Logo() {
+  return (
+    <Box className="logo">
+      <img src="/logo.svg" alt="Promptfoo logo" /> <span>promptfoo</span>
+    </Box>
+  );
+}

package/src/web/client/src/NavBar.css

@@ -0,0 +1,3 @@
+nav {
+  margin-bottom: 1rem;
+}

package/src/web/client/src/NavBar.tsx

@@ -0,0 +1,11 @@
+import Logo from './Logo';
+
+import './NavBar.css';
+
+export default function NavBar() {
+  return (
+    <nav>
+      <Logo />
+    </nav>
+  );
+}

package/src/web/client/src/ResultsTable.css

@@ -0,0 +1,133 @@
+* {
+  box-sizing: border-box;
+}
+
+html {
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Helvetica, Arial, sans-serif,
+    'Apple Color Emoji', 'Segoe UI Emoji', 'Segoe UI Symbol';
+  font-size: 16px;
+  background-color: var(--background-color);
+  color: var(--text-color);
+}
+
+table,
+.divTable {
+  border: 1px solid var(--table-border-color);
+  border-collapse: collapse;
+  width: 100%;
+
+  margin: 1rem 0;
+  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
+}
+
+.tr {
+  display: flex;
+}
+
+tr,
+.tr {
+  width: fit-content;
+}
+
+tr:hover,
+.tr:hover {
+  background-color: rgba(0, 0, 0, 0.05);
+}
+
+th,
+.th,
+td,
+.td {
+  position: relative;
+  box-shadow: inset 0 0 0 1px var(--border-color);
+  word-break: break-all;
+  vertical-align: top;
+
+  padding: 1.5rem;
+}
+
+th,
+.th {
+  padding: 1rem;
+  position: relative;
+  text-align: center;
+  font-weight: semi-bold;
+}
+
+tr .cell {
+}
+
+tr .cell-rating {
+  visibility: hidden;
+  position: absolute;
+  bottom: 1.25rem;
+  right: -1rem;
+  line-height: 0;
+  font-size: 1.75rem;
+}
+
+tr:hover .cell-rating {
+  visibility: visible;
+}
+
+tr .cell-rating .rating {
+  cursor: pointer;
+  margin-right: 1rem;
+}
+
+th .smalltext {
+  visibility: hidden;
+  font-weight: normal;
+  font-size: 0.75rem;
+  color: var(--smalltext-color);
+}
+
+th:hover .smalltext {
+  visibility: visible;
+}
+
+td,
+.td {
+}
+
+td .status {
+  margin-bottom: 0.5rem;
+  font-weight: bold;
+}
+
+td .pass {
+  color: var(--pass-color);
+}
+
+td .fail {
+  color: var(--fail-color);
+}
+
+.resizer {
+  position: absolute;
+  right: 0;
+  top: 0;
+  height: 100%;
+  width: 5px;
+  cursor: col-resize;
+  user-select: none;
+  touch-action: none;
+
+  background: var(--text-color);
+  opacity: 0.5;
+}
+
+.resizer.isResizing {
+  background: var(--text-color);
+  opacity: 1;
+}
+
+@media (hover: hover) {
+  .resizer {
+    opacity: 0;
+  }
+
+  *:hover > .resizer {
+    opacity: 1;
+  }
+}

package/src/web/client/src/ResultsTable.tsx

@@ -0,0 +1,261 @@
+import * as React from 'react';
+
+import './index.css';
+
+import invariant from 'tiny-invariant';
+import {
+  createColumnHelper,
+  flexRender,
+  getCoreRowModel,
+  useReactTable,
+} from '@tanstack/react-table';
+
+import { useStore } from './store.js';
+
+import type { CellContext, VisibilityState } from '@tanstack/table-core';
+
+import type { EvalRow } from './types.js';
+
+import './ResultsTable.css';
+
+interface TruncatedTextProps {
+  text: string;
+  maxLength: number;
+}
+
+function TruncatedText({ text, maxLength }: TruncatedTextProps) {
+  const [isTruncated, setIsTruncated] = React.useState<boolean>(true);
+
+  const toggleTruncate = () => {
+    setIsTruncated(!isTruncated);
+  };
+
+  const renderTruncatedText = () => {
+    if (text.length <= maxLength) {
+      return text;
+    }
+    if (isTruncated) {
+      return (
+        <>
+          <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
+            {text.substring(0, maxLength)} ...
+          </span>
+        </>
+      );
+    } else {
+      return (
+        <>
+          <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
+            {text}
+          </span>
+        </>
+      );
+    }
+  };
+
+  return <div>{renderTruncatedText()}</div>;
+}
+
+interface PromptOutputProps {
+  text: string;
+  maxTextLength: number;
+  rowIndex: number;
+  promptIndex: number;
+  onRating: (rowIndex: number, promptIndex: number, isPass: boolean) => void;
+}
+
+function PromptOutput({ text, maxTextLength, rowIndex, promptIndex, onRating }: PromptOutputProps) {
+  const isPass = text.startsWith('[PASS] ');
+  const isFail = text.startsWith('[FAIL] ');
+  let chunks: string[] = [];
+  if (isPass) {
+    text = text.substring(7);
+  } else if (isFail) {
+    if (text.includes('---')) {
+      chunks = text.split('---');
+      text = chunks.slice(1).join('---');
+    } else {
+      chunks = ['[FAIL]'];
+      if (text.startsWith('[FAIL] ')) {
+        text = text.substring(7);
+      }
+    }
+  }
+
+  const handleClick = (isPass: boolean) => {
+    onRating(rowIndex, promptIndex, isPass);
+  };
+
+  return (
+    <>
+      <div className="cell">
+        {isPass && <div className="status pass">[PASS]</div>}
+        {isFail && <div className="status fail">{chunks[0]}</div>}{' '}
+        <TruncatedText text={text} maxLength={maxTextLength} />
+      </div>
+      <div className="cell-rating">
+        <span className="rating" onClick={() => handleClick(true)}>
+          👍
+        </span>
+        <span className="rating" onClick={() => handleClick(false)}>
+          👎
+        </span>
+      </div>
+    </>
+  );
+}
+
+function TableHeader({ text, maxLength, smallText }: TruncatedTextProps & { smallText: string }) {
+  return (
+    <div>
+      <TruncatedText text={text} maxLength={maxLength} />
+      <span className="smalltext">{smallText}</span>
+    </div>
+  );
+}
+
+interface ResultsViewProps {
+  maxTextLength: number;
+  columnVisibility: VisibilityState;
+}
+
+export default function ResultsTable({ maxTextLength, columnVisibility }: ResultsViewProps) {
+  const { table, setTable } = useStore();
+  invariant(table, 'Table should be defined');
+  const { head, body } = table;
+  // TODO(ian): Correctly plumb through the results instead of parsing the string.
+  const numGood = head.prompts.map((_, idx) =>
+    body.reduce((acc, row) => {
+      return acc + (row.outputs[idx].startsWith('[PASS]') ? 1 : 0);
+    }, 0),
+  );
+
+  const handleRating = (rowIndex: number, promptIndex: number, isPass: boolean) => {
+    const updatedData = [...body];
+    const updatedRow = { ...updatedData[rowIndex] };
+    const updatedOutputs = [...updatedRow.outputs];
+    const updatedOutput = isPass
+      ? `[PASS] ${updatedOutputs[promptIndex].replace(/^\[(PASS|FAIL)\] /, '')}`
+      : `[FAIL] ${updatedOutputs[promptIndex].replace(/^\[(PASS|FAIL)\] /, '')}`;
+    updatedOutputs[promptIndex] = updatedOutput;
+    updatedRow.outputs = updatedOutputs;
+    updatedData[rowIndex] = updatedRow;
+    setTable({
+      head,
+      body: updatedData,
+    });
+  };
+
+  const columnHelper = createColumnHelper<EvalRow>();
+  const columns = [
+    columnHelper.group({
+      id: 'prompts',
+      header: () => <span>Prompts</span>,
+      columns: head.prompts.map((prompt, idx) =>
+        columnHelper.accessor((row: EvalRow) => row.outputs[idx], {
+          id: `Prompt ${idx + 1}`,
+          header: () => (
+            <>
+              <TableHeader
+                smallText={`Prompt ${idx + 1}`}
+                text={prompt}
+                maxLength={maxTextLength}
+              />
+              {numGood[idx]} / {body.length} 👍
+            </>
+          ),
+          cell: (info: CellContext<EvalRow, string>) => (
+            <PromptOutput
+              text={info.getValue()}
+              maxTextLength={maxTextLength}
+              rowIndex={info.row.index}
+              promptIndex={idx}
+              onRating={handleRating}
+            />
+          ),
+        }),
+      ),
+    }),
+    columnHelper.group({
+      id: 'vars',
+      header: () => <span>Variables</span>,
+      columns: head.vars.map((varName, idx) =>
+        columnHelper.accessor((row: EvalRow) => row.vars[idx], {
+          id: `Variable ${idx + 1}`,
+          header: () => (
+            <TableHeader
+              smallText={`Variable ${idx + 1}`}
+              text={varName}
+              maxLength={maxTextLength}
+            />
+          ),
+          cell: (info: CellContext<EvalRow, string>) => (
+            <TruncatedText text={info.getValue()} maxLength={maxTextLength} />
+          ),
+        }),
+      ),
+    }),
+  ];
+
+  const reactTable = useReactTable({
+    data: body,
+    columns,
+    columnResizeMode: 'onChange',
+    getCoreRowModel: getCoreRowModel(),
+
+    state: {
+      columnVisibility,
+    },
+  });
+
+  return (
+    <table>
+      <thead>
+        {reactTable.getHeaderGroups().map((headerGroup) => (
+          <tr key={headerGroup.id}>
+            {headerGroup.headers.map((header) => (
+              <th
+                {...{
+                  key: header.id,
+                  colSpan: header.colSpan,
+                  style: {
+                    width: header.getSize(),
+                  },
+                }}
+              >
+                {header.isPlaceholder
+                  ? null
+                  : flexRender(header.column.columnDef.header, header.getContext())}
+                <div
+                  {...{
+                    onMouseDown: header.getResizeHandler(),
+                    onTouchStart: header.getResizeHandler(),
+                    className: `resizer ${header.column.getIsResizing() ? 'isResizing' : ''}`,
+                  }}
+                />
+              </th>
+            ))}
+          </tr>
+        ))}
+      </thead>
+      <tbody>
+        {reactTable.getRowModel().rows.map((row) => (
+          <tr key={row.id}>
+            {row.getVisibleCells().map((cell) => (
+              <td
+                {...{
+                  key: cell.id,
+                  style: {
+                    width: cell.column.getSize(),
+                  },
+                }}
+              >
+                {flexRender(cell.column.columnDef.cell, cell.getContext())}
+              </td>
+            ))}
+          </tr>
+        ))}
+      </tbody>
+    </table>
+  );
+}

package/src/web/client/src/ResultsView.tsx

@@ -0,0 +1,110 @@
+import * as React from 'react';
+
+import invariant from 'tiny-invariant';
+import Box from '@mui/material/Box';
+import Paper from '@mui/material/Box';
+import Stack from '@mui/material/Stack';
+import Slider from '@mui/material/Slider';
+import Typography from '@mui/material/Typography';
+import OutlinedInput from '@mui/material/OutlinedInput';
+import InputLabel from '@mui/material/InputLabel';
+import MenuItem from '@mui/material/MenuItem';
+import FormControl from '@mui/material/FormControl';
+import ListItemText from '@mui/material/ListItemText';
+import Select, { SelectChangeEvent } from '@mui/material/Select';
+import Checkbox from '@mui/material/Checkbox';
+
+import ResultsTable from './ResultsTable.js';
+import { useStore } from './store.js';
+
+import type { VisibilityState } from '@tanstack/table-core';
+
+export default function ResultsView() {
+  const { table } = useStore();
+  const [maxTextLength, setMaxTextLength] = React.useState(250);
+  const [columnVisibility, setColumnVisibility] = React.useState<VisibilityState>({});
+  const [selectedColumns, setSelectedColumns] = React.useState<string[]>([]);
+
+  invariant(table, 'Table data must be loaded before rendering ResultsView');
+  const { head } = table;
+
+  const handleChange = (event: SelectChangeEvent<typeof selectedColumns>) => {
+    const {
+      target: { value },
+    } = event;
+    setSelectedColumns(typeof value === 'string' ? value.split(',') : value);
+
+    const allColumns = [
+      ...head.prompts.map((_, idx) => `Prompt ${idx + 1}`),
+      ...head.vars.map((_, idx) => `Variable ${idx + 1}`),
+    ];
+    const newColumnVisibility: VisibilityState = {};
+    allColumns.forEach((col) => {
+      newColumnVisibility[col] = (typeof value === 'string' ? value.split(',') : value).includes(
+        col,
+      );
+    });
+    setColumnVisibility(newColumnVisibility);
+  };
+
+  const columnData = [
+    ...head.prompts.map((_, idx) => ({
+      value: `Prompt ${idx + 1}`,
+      label: `Prompt ${idx + 1}`,
+      group: 'Prompts',
+    })),
+    ...head.vars.map((_, idx) => ({
+      value: `Variable ${idx + 1}`,
+      label: `Variable ${idx + 1}`,
+      group: 'Variables',
+    })),
+  ];
+
+  // Set all columns as selected by default
+  React.useEffect(() => {
+    setSelectedColumns([
+      ...head.prompts.map((_, idx) => `Prompt ${idx + 1}`),
+      ...head.vars.map((_, idx) => `Variable ${idx + 1}`),
+    ]);
+  }, [head]);
+
+  return (
+    <div>
+      <Paper py="md">
+        <Stack direction="row" spacing={2} alignItems="center">
+          <Box>
+            <FormControl sx={{ m: 1, minWidth: 300 }} size="small">
+              <InputLabel id="visible-columns-label">Visible columns</InputLabel>
+              <Select
+                labelId="visible-columns-label"
+                id="visible-columns"
+                multiple
+                value={selectedColumns}
+                onChange={handleChange}
+                input={<OutlinedInput label="Visible columns" />}
+                renderValue={(selected: string[]) => selected.join(', ')}
+              >
+                {columnData.map((column) => (
+                  <MenuItem dense key={column.value} value={column.value}>
+                    <Checkbox checked={selectedColumns.indexOf(column.value) > -1} />
+                    <ListItemText primary={column.label} />
+                  </MenuItem>
+                ))}
+              </Select>
+            </FormControl>
+          </Box>
+          <Box>
+            <Typography mt={2}>Max text length: {maxTextLength}</Typography>
+            <Slider
+              min={25}
+              max={1000}
+              value={maxTextLength}
+              onChange={(_, val: number | number[]) => setMaxTextLength(val as number)}
+            />
+          </Box>
+        </Stack>
+      </Paper>
+      <ResultsTable maxTextLength={maxTextLength} columnVisibility={columnVisibility} />
+    </div>
+  );
+}

package/src/web/client/src/index.css

@@ -0,0 +1,35 @@
+:root {
+  font-family: system-ui, Avenir, Helvetica, Arial, sans-serif;
+
+  font-synthesis: none;
+  text-rendering: optimizeLegibility;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  -webkit-text-size-adjust: 100%;
+
+  /* Light mode colors */
+  --background-color: #ffffff;
+  --text-color: #404040;
+  --border-color: lightgray;
+  --table-border-color: lightgray;
+  --pass-color: green;
+  --fail-color: #ad0000;
+  --smalltext-color: gray;
+}
+
+/* Dark mode colors */
+@media (prefers-color-scheme: dark) {
+  :root {
+    --background-color: #1a1a1a;
+    --text-color: #f0f0f0;
+    --border-color: #444444;
+    --table-border-color: #444444;
+    --pass-color: #4caf50;
+    --fail-color: #f44336;
+    --smalltext-color: #888888;
+  }
+}
+
+html {
+  font-size: calc(14px + (18 - 14) * ((100vw - 300px) / (1600 - 300)));
+}

package/src/web/client/src/main.tsx

@@ -0,0 +1,10 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App.tsx';
+import './index.css';
+
+ReactDOM.createRoot(document.getElementById('root') as HTMLElement).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+);

package/src/web/client/src/store.ts

@@ -0,0 +1,13 @@
+import create from 'zustand';
+
+import type { EvalTable } from './types.js';
+
+interface TableState {
+  table: EvalTable | null;
+  setTable: (table: EvalTable | null) => void;
+}
+
+export const useStore = create<TableState>((set) => ({
+  table: null,
+  setTable: (table: EvalTable | null) => set(() => ({ table })),
+}));

package/src/web/client/src/types.ts

@@ -0,0 +1,14 @@
+export type EvalHead = {
+  prompts: string[];
+  vars: string[];
+};
+
+export type EvalRow = {
+  outputs: string[]; // var inputs
+  vars: string[]; // model outputs
+};
+
+export type EvalTable = {
+  head: EvalHead;
+  body: EvalRow[];
+};

package/src/web/client/src/vite-env.d.ts

@@ -0,0 +1 @@
+/// <reference types="vite/client" />

package/src/web/client/tsconfig.json

@@ -0,0 +1,24 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "lib": ["DOM", "DOM.Iterable", "ESNext"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+
+    /* Bundler mode */
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx",
+
+    /* Linting */
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true
+  },
+  "include": ["src"],
+  "references": [{ "path": "./tsconfig.node.json" }]
+}