promptfoo 0.20.1 → 0.21.0

package/src/web/nextui/src/app/eval/ResultsCharts.tsx

@@ -1,364 +0,0 @@
-import React, { useRef, useEffect, useState } from 'react';
-import {
-  Chart,
-  BarController,
-  LineController,
-  ScatterController,
-  CategoryScale,
-  LinearScale,
-  BarElement,
-  LineElement,
-  PointElement,
-  Tooltip,
-  Colors,
-} from 'chart.js';
-import Select from '@mui/material/Select';
-import MenuItem from '@mui/material/MenuItem';
-import FormControl from '@mui/material/FormControl';
-import Dialog from '@mui/material/Dialog';
-import DialogTitle from '@mui/material/DialogTitle';
-import DialogContent from '@mui/material/DialogContent';
-import { useTheme } from '@mui/material/styles';
-import Paper from '@mui/material/Paper';
-import IconButton from '@mui/material/IconButton';
-import CloseIcon from '@mui/icons-material/Close';
-import { ErrorBoundary } from 'react-error-boundary';
-
-import { useStore } from './store';
-
-import type { VisibilityState } from '@tanstack/table-core';
-import type { EvalTable } from './types';
-
-interface ResultsChartsProps {
-  columnVisibility: VisibilityState;
-}
-
-interface ChartProps {
-  table: EvalTable;
-}
-
-const COLOR_PALETTE = [
-  '#fd7f6f',
-  '#7eb0d5',
-  '#b2e061',
-  '#bd7ebe',
-  '#ffb55a',
-  '#ffee65',
-  '#beb9db',
-  '#fdcce5',
-  '#8bd3c7',
-];
-
-Chart.register(
-  BarController,
-  LineController,
-  ScatterController,
-  CategoryScale,
-  LinearScale,
-  BarElement,
-  LineElement,
-  PointElement,
-  Tooltip,
-  Colors,
-);
-
-function HistogramChart({ table }: ChartProps) {
-  const histogramCanvasRef = useRef(null);
-  const histogramChartInstance = useRef<Chart | null>(null);
-
-  useEffect(() => {
-    if (!histogramCanvasRef.current) {
-      return;
-    }
-
-    if (histogramChartInstance.current) {
-      histogramChartInstance.current.destroy();
-    }
-
-    // Calculate bins and their counts
-    const scores = table.body.flatMap((row) => row.outputs.map((output) => output.score));
-    const maxScore = Math.max(...scores);
-    const minScore = Math.min(...scores);
-    const range = Math.ceil(maxScore) - Math.floor(minScore); // Adjust the range to be between whole numbers
-    const binSize = range / 10; // Define the size of each bin
-    const bins = Array.from({ length: 11 }, (_, i) =>
-      parseFloat((Math.floor(minScore) + i * binSize).toFixed(2)),
-    );
-
-    const datasets = table.head.prompts.map((prompt, promptIdx) => {
-      const scores = table.body.flatMap((row) => row.outputs[promptIdx].score);
-      const counts = bins.map(
-        (bin) => scores.filter((score) => score >= bin && score < bin + binSize).length,
-      );
-      return {
-        label: `Prompt ${promptIdx + 1}`,
-        data: counts,
-        backgroundColor: COLOR_PALETTE[promptIdx % COLOR_PALETTE.length],
-      };
-    });
-
-    histogramChartInstance.current = new Chart(histogramCanvasRef.current, {
-      type: 'bar',
-      data: {
-        labels: bins,
-        datasets,
-      },
-      options: {
-        animation: false,
-        plugins: {
-          title: {
-            display: true,
-            text: 'Score Distribution',
-          },
-          legend: {
-            display: false,
-          },
-          tooltip: {
-            callbacks: {
-              title: function (context) {
-                const datasetIndex = context[0].datasetIndex;
-                return `Prompt ${datasetIndex + 1}`;
-              },
-              label: function (context) {
-                const labelIndex = context.dataIndex;
-                const lowerBound = bins[labelIndex];
-                const upperBound = bins[labelIndex + 1];
-                if (!upperBound) {
-                  return `${lowerBound} <= score`;
-                }
-                return `${lowerBound} <= score < ${upperBound}`;
-              },
-            },
-          },
-        },
-      },
-    });
-  }, [table]);
-
-  return <canvas ref={histogramCanvasRef} style={{ maxHeight: '300px' }}></canvas>;
-}
-
-function PassRateChart({ table }: ChartProps) {
-  const passRateCanvasRef = useRef(null);
-  const passRateChartInstance = useRef<Chart | null>(null);
-
-  useEffect(() => {
-    if (!passRateCanvasRef.current) {
-      return;
-    }
-
-    if (passRateChartInstance.current) {
-      passRateChartInstance.current.destroy();
-    }
-
-    const datasets = table.head.prompts.map((prompt, promptIdx) => {
-      const outputs = table.body.flatMap((row) => row.outputs[promptIdx]);
-      const passCount = outputs.filter((output) => output.pass).length;
-      const passRate = (passCount / outputs.length) * 100;
-      return {
-        label: `Prompt ${promptIdx + 1}`,
-        data: [passRate],
-        backgroundColor: COLOR_PALETTE[promptIdx % COLOR_PALETTE.length],
-      };
-    });
-
-    passRateChartInstance.current = new Chart(passRateCanvasRef.current, {
-      type: 'bar',
-      data: {
-        labels: ['Pass Rate (%)'],
-        datasets,
-      },
-      options: {
-        animation: false,
-        plugins: {
-          title: {
-            display: true,
-            text: 'Pass rate',
-          },
-          legend: {
-            display: true,
-          },
-        },
-      },
-    });
-  }, [table]);
-
-  return <canvas ref={passRateCanvasRef} style={{ maxHeight: '300px' }}></canvas>;
-}
-
-function ScatterChart({ table }: ChartProps) {
-  const scatterCanvasRef = useRef(null);
-  const scatterChartInstance = useRef<Chart | null>(null);
-  const [xAxisPrompt, setXAxisPrompt] = useState(0);
-  const [yAxisPrompt, setYAxisPrompt] = useState(1);
-  const [open, setOpen] = useState(false);
-
-  useEffect(() => {
-    if (!scatterCanvasRef.current) {
-      return;
-    }
-
-    if (scatterChartInstance.current) {
-      scatterChartInstance.current.destroy();
-    }
-
-    const scores = table.body.flatMap((row) => row.outputs.map((output) => output.score));
-    const minScore = Math.min(...scores);
-    const maxScore = Math.max(...scores);
-
-    const data = table.body.map((row) => {
-      const prompt1Score = row.outputs[xAxisPrompt].score;
-      const prompt2Score = row.outputs[yAxisPrompt].score;
-      let backgroundColor;
-      if (prompt2Score > prompt1Score) {
-        backgroundColor = 'green';
-      } else if (prompt2Score < prompt1Score) {
-        backgroundColor = 'red';
-      } else {
-        backgroundColor = 'gray';
-      }
-      return {
-        x: prompt1Score,
-        y: prompt2Score,
-        backgroundColor,
-      };
-    });
-
-    scatterChartInstance.current = new Chart(scatterCanvasRef.current, {
-      type: 'scatter',
-      data: {
-        datasets: [
-          {
-            data,
-            backgroundColor: data.map((point) => point.backgroundColor),
-          },
-          {
-            type: 'line',
-            data: [
-              // @ts-ignore: types seem wrong, it wants backgroundColor
-              { x: minScore, y: minScore },
-              // @ts-ignore: types seem wrong, it wants backgroundColor
-              { x: maxScore, y: maxScore },
-            ],
-            borderColor: 'gray',
-            borderWidth: 1,
-            borderDash: [5, 5],
-            pointRadius: 0,
-          },
-        ],
-      },
-      options: {
-        animation: false,
-        plugins: {
-          legend: {
-            display: false,
-          },
-          tooltip: {
-            callbacks: {
-              label: function (tooltipItem) {
-                const row = table.body[tooltipItem.dataIndex];
-                let prompt1Text = row.outputs[0].text;
-                let prompt2Text = row.outputs[1].text;
-                if (prompt1Text.length > 30) {
-                  prompt1Text = prompt1Text.substring(0, 30) + '...';
-                }
-                if (prompt2Text.length > 30) {
-                  prompt2Text = prompt2Text.substring(0, 30) + '...';
-                }
-                return `Output 1: ${prompt1Text}\nOutput 2: ${prompt2Text}`;
-              },
-            },
-          },
-        },
-        scales: {
-          x: {
-            title: {
-              display: true,
-              text: `Prompt ${xAxisPrompt + 1} Score`,
-            },
-          },
-          y: {
-            title: {
-              display: true,
-              text: `Prompt ${yAxisPrompt + 1} Score`,
-            },
-          },
-        },
-      },
-    });
-  }, [table, xAxisPrompt, yAxisPrompt]);
-
-  return (
-    <>
-      <Dialog open={open} onClose={() => setOpen(false)}>
-        <DialogTitle>Compare prompt outputs</DialogTitle>
-        <DialogContent>
-          <FormControl sx={{ m: 1, minWidth: 120 }}>
-            <Select value={xAxisPrompt} onChange={(e) => setXAxisPrompt(Number(e.target.value))}>
-              {table.head.prompts.map((prompt, idx) => (
-                <MenuItem key={idx} value={idx}>
-                  Prompt {idx + 1}
-                </MenuItem>
-              ))}
-            </Select>
-          </FormControl>
-          <FormControl sx={{ m: 1, minWidth: 120 }}>
-            <Select value={yAxisPrompt} onChange={(e) => setYAxisPrompt(Number(e.target.value))}>
-              {table.head.prompts.map((prompt, idx) => (
-                <MenuItem key={idx} value={idx}>
-                  Prompt {idx + 1}
-                </MenuItem>
-              ))}
-            </Select>
-          </FormControl>
-        </DialogContent>
-      </Dialog>
-      <canvas
-        ref={scatterCanvasRef}
-        style={{ maxHeight: '300px', cursor: 'pointer' }}
-        onClick={() => setOpen(true)}
-      ></canvas>
-    </>
-  );
-}
-
-export default function ResultsCharts({ columnVisibility }: ResultsChartsProps) {
-  const theme = useTheme();
-  Chart.defaults.color = theme.palette.mode === 'dark' ? '#aaa' : '#666';
-  const [showCharts, setShowCharts] = useState(true);
-
-  const { table } = useStore();
-  if (!table || !showCharts || table.head.prompts.length < 2) {
-    return null;
-  }
-
-  const scores = table.body.flatMap((row) => row.outputs.map((output) => output.score));
-  if (new Set(scores).size === 1) {
-    // All scores are the same, charts not useful.
-    return null;
-  }
-
-  return (
-    <ErrorBoundary fallback={null}>
-      <Paper style={{ position: 'relative', padding: theme.spacing(3) }}>
-        <IconButton
-          style={{ position: 'absolute', right: 0, top: 0 }}
-          onClick={() => setShowCharts(false)}
-        >
-          <CloseIcon />
-        </IconButton>
-        <div style={{ display: 'flex', justifyContent: 'space-between', width: '100%' }}>
-          <div style={{ width: '33%' }}>
-            <PassRateChart table={table} />
-          </div>
-          <div style={{ width: '33%' }}>
-            <HistogramChart table={table} />
-          </div>
-          <div style={{ width: '33%' }}>
-            <ScatterChart table={table} />
-          </div>
-        </div>
-      </Paper>
-    </ErrorBoundary>
-  );
-}

package/src/web/nextui/src/app/eval/ResultsTable.css

@@ -1,179 +0,0 @@
-table,
-.divTable {
-  border: 1px solid var(--table-border-color);
-  border-collapse: collapse;
-  width: 100%;
-
-  margin: 1rem 0;
-  box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
-}
-
-ins {
-  background-color: var(--insert-highlight-color);
-  text-decoration: none;
-}
-
-del {
-  background-color: var(--delete-highlight-color);
-  text-decoration: strikethrough;
-}
-
-.tr {
-  display: flex;
-}
-
-tr,
-.tr {
-  width: fit-content;
-}
-
-tr:hover,
-.tr:hover {
-  background-color: rgba(0, 0, 0, 0.05);
-}
-
-th,
-.th,
-td,
-.td {
-  position: relative;
-  box-shadow: inset 0 0 0 1px var(--border-color);
-  vertical-align: top;
-
-  padding: 1.5rem;
-}
-
-th.variable,
-.th.variable,
-td.variable,
-.td.variable {
-  background-color: var(--variable-background-color);
-}
-
-tr.header {
-  background-color: var(--header-background-color);
-}
-
-th,
-.th {
-  padding: 1rem;
-  position: relative;
-  text-align: center;
-  vertical-align: bottom;
-}
-
-th .action {
-  cursor: pointer;
-  margin-left: 0.5rem;
-}
-
-tr .cell {
-}
-
-tr .cell-actions {
-  display: flex;
-  gap: 0.5rem;
-  visibility: hidden;
-  position: absolute;
-  bottom: 1.25rem;
-  right: 0;
-  line-height: 0;
-  font-size: 1.75rem;
-}
-
-tr .cell-detail {
-  visibility: hidden;
-  position: absolute;
-  bottom: 0.25rem;
-  margin-top: 1rem;
-  font-size: 0.75rem;
-  color: #888;
-}
-
-tr:hover .cell-actions,
-tr:hover .cell-detail {
-  visibility: visible;
-}
-
-tr .cell-actions .action {
-  cursor: pointer;
-}
-
-th .smalltext {
-  visibility: hidden;
-  font-weight: normal;
-  font-size: 0.75rem;
-  color: var(--smalltext-color);
-}
-
-th:hover .smalltext {
-  visibility: visible;
-}
-
-th .summary {
-  font-weight: normal;
-  font-size: 0.8rem;
-  padding: 0.25rem;
-}
-
-th .summary.highlight {
-  background-color: var(--success-background-color);
-}
-
-td,
-.td {
-}
-
-td .status {
-  margin-bottom: 0.5rem;
-  font-weight: bold;
-}
-
-td .score {
-  font-weight: normal;
-}
-
-td .pass {
-  color: var(--pass-color);
-}
-
-td .fail {
-  color: var(--fail-color);
-}
-
-.first-prompt-col {
-  border-left: 2px solid #888;
-}
-
-.first-prompt-row {
-  border-top: 2px solid #888;
-}
-
-.resizer {
-  position: absolute;
-  right: 0;
-  top: 0;
-  height: 100%;
-  width: 5px;
-  cursor: col-resize;
-  user-select: none;
-  touch-action: none;
-
-  background: var(--text-color);
-  opacity: 0.5;
-}
-
-.resizer.isResizing {
-  background: var(--text-color);
-  opacity: 1;
-}
-
-@media (hover: hover) {
-  .resizer {
-    opacity: 0;
-  }
-
-  *:hover > .resizer {
-    opacity: 1;
-  }
-}