promptfoo 0.15.0 → 0.17.0

package/src/web/client/src/EvalOutputPromptDialog.tsx

@@ -0,0 +1,61 @@
+import { useState, useEffect } from 'react';
+import Button from '@mui/material/Button';
+import Dialog from '@mui/material/Dialog';
+import DialogActions from '@mui/material/DialogActions';
+import DialogContent from '@mui/material/DialogContent';
+import DialogTitle from '@mui/material/DialogTitle';
+import TextareaAutosize from '@mui/base/TextareaAutosize';
+import IconButton from '@mui/material/IconButton';
+import ContentCopyIcon from '@mui/icons-material/ContentCopy';
+import CheckIcon from '@mui/icons-material/Check';
+
+interface EvalOutputPromptDialogProps {
+  open: boolean;
+  onClose: () => void;
+  prompt: string;
+  output?: string;
+}
+
+export default function EvalOutputPromptDialog({
+  open,
+  onClose,
+  prompt,
+  output,
+}: EvalOutputPromptDialogProps) {
+  const [copied, setCopied] = useState(false);
+
+  useEffect(() => {
+    setCopied(false);
+  }, [prompt]);
+
+  const copyToClipboard = async (text: string) => {
+    await navigator.clipboard.writeText(text);
+    setCopied(true);
+  };
+
+  return (
+    <Dialog open={open} onClose={onClose} fullWidth maxWidth="lg">
+      <DialogTitle>Prompt</DialogTitle>
+      <DialogContent>
+        <TextareaAutosize readOnly value={prompt} style={{ width: '100%' }} />
+        <IconButton
+          onClick={() => copyToClipboard(prompt)}
+          style={{ position: 'absolute', right: '10px', top: '10px' }}
+        >
+          {copied ? <CheckIcon /> : <ContentCopyIcon />}
+        </IconButton>
+      </DialogContent>
+      {output && (
+        <>
+          <DialogTitle>Output</DialogTitle>
+          <DialogContent>
+            <TextareaAutosize readOnly value={output} style={{ width: '100%' }} />
+          </DialogContent>
+        </>
+      )}
+      <DialogActions>
+        <Button onClick={onClose}>Close</Button>
+      </DialogActions>
+    </Dialog>
+  );
+}

package/src/web/client/src/ResultsTable.css

@@ -64,10 +64,17 @@ th,
   vertical-align: bottom;
 }
 
+th .action {
+  cursor: pointer;
+  margin-left: 0.5rem;
+}
+
 tr .cell {
 }
 
-tr .cell-rating {
+tr .cell-actions {
+  display: flex;
+  gap: 0.5rem;
   visibility: hidden;
   position: absolute;
   bottom: 1.25rem;
@@ -76,18 +83,14 @@ tr .cell-rating {
   font-size: 1.75rem;
 }
 
-tr:hover .cell-rating {
+tr:hover .cell-actions {
   visibility: visible;
 }
 
-tr .cell-rating .rating {
+tr .cell-actions .action {
   cursor: pointer;
 }
 
-tr .cell-rating .rating:first-child {
-  margin-right: 0.5rem;
-}
-
 th .smalltext {
   visibility: hidden;
   font-weight: normal;

package/src/web/client/src/ResultsTable.tsx

@@ -16,6 +16,8 @@ import { useStore } from './store.js';
 
 import type { CellContext, VisibilityState } from '@tanstack/table-core';
 
+import EvalOutputPromptDialog from './EvalOutputPromptDialog';
+
 import type { EvalRow, EvalRowOutput, FilterMode } from './types.js';
 
 import './ResultsTable.css';
@@ -54,7 +56,7 @@ interface TruncatedTextProps {
 
 function TruncatedText({ text: rawText, maxLength }: TruncatedTextProps) {
   const [isTruncated, setIsTruncated] = React.useState<boolean>(true);
-  const text = String(rawText);
+  const text = typeof rawText === 'string' ? rawText : JSON.stringify(rawText);
 
   const toggleTruncate = () => {
     setIsTruncated(!isTruncated);
@@ -62,28 +64,24 @@ function TruncatedText({ text: rawText, maxLength }: TruncatedTextProps) {
 
   const renderTruncatedText = () => {
     if (text.length <= maxLength) {
-      return text;
+      return <span>text</span>;
     }
     if (isTruncated) {
       return (
-        <>
-          <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
-            {text.substring(0, maxLength)} ...
-          </span>
-        </>
+        <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
+          {text.substring(0, maxLength)} ...
+        </span>
       );
     } else {
       return (
-        <>
-          <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
-            {text}
-          </span>
-        </>
+        <span style={{ cursor: 'pointer' }} onClick={toggleTruncate}>
+          {text}
+        </span>
       );
     }
   };
 
-  return <div>{renderTruncatedText()}</div>;
+  return renderTruncatedText();
 }
 
 interface PromptOutputProps {
@@ -94,14 +92,21 @@ interface PromptOutputProps {
   onRating: (rowIndex: number, promptIndex: number, isPass: boolean) => void;
 }
 
-function PromptOutput({
+function EvalOutputCell({
   output,
   maxTextLength,
   rowIndex,
   promptIndex,
   onRating,
 }: PromptOutputProps) {
-  let text = String(output.text);
+  const [openPrompt, setOpen] = React.useState(false);
+  const handlePromptOpen = () => {
+    setOpen(true);
+  };
+  const handlePromptClose = () => {
+    setOpen(false);
+  };
+  let text = typeof output.text === 'string' ? output.text : JSON.stringify(output.text);
   let chunks: string[] = [];
   if (!output.pass && text.includes('---')) {
     // TODO(ian): Plumb through failure message instead of parsing it out.
@@ -113,6 +118,7 @@ function PromptOutput({
     onRating(rowIndex, promptIndex, isPass);
   };
 
+  // TODO(ian): output.prompt check for backwards compatibility, remove after 0.17.0
   return (
     <>
       <div className="cell">
@@ -128,11 +134,24 @@ function PromptOutput({
         )}{' '}
         <TruncatedText text={text} maxLength={maxTextLength} />
       </div>
-      <div className="cell-rating">
-        <span className="rating" onClick={() => handleClick(true)}>
+      <div className="cell-actions">
+        {output.prompt && (
+          <>
+            <span className="action" onClick={handlePromptOpen}>
+              🔎
+            </span>
+            <EvalOutputPromptDialog
+              open={openPrompt}
+              onClose={handlePromptClose}
+              prompt={output.prompt}
+              output={text}
+            />
+          </>
+        )}
+        <span className="action" onClick={() => handleClick(true)}>
           👍
         </span>
-        <span className="rating" onClick={() => handleClick(false)}>
+        <span className="action" onClick={() => handleClick(false)}>
           👎
         </span>
       </div>
@@ -140,11 +159,35 @@ function PromptOutput({
   );
 }
 
-function TableHeader({ text, maxLength, smallText }: TruncatedTextProps & { smallText: string }) {
+function TableHeader({
+  text,
+  maxLength,
+  smallText,
+  expandedText,
+}: TruncatedTextProps & { smallText: string; expandedText?: string }) {
+  const [openPrompt, setOpen] = React.useState(false);
+  const handlePromptOpen = () => {
+    setOpen(true);
+  };
+  const handlePromptClose = () => {
+    setOpen(false);
+  };
   return (
     <div>
       <TruncatedText text={text} maxLength={maxLength} />
-      <span className="smalltext">{smallText}</span>
+      {expandedText && (
+        <>
+          <span className="action" onClick={handlePromptOpen}>
+            🔎
+          </span>
+          <EvalOutputPromptDialog
+            open={openPrompt}
+            onClose={handlePromptClose}
+            prompt={expandedText}
+          />
+        </>
+      )}
+      <div className="smalltext">{smallText}</div>
     </div>
   );
 }
@@ -199,21 +242,26 @@ export default function ResultsTable({
       id: 'vars',
       header: () => <span>Variables</span>,
       columns: head.vars.map((varName, idx) =>
-        columnHelper.accessor((row: EvalRow) => row.vars[idx], {
-          id: `Variable ${idx + 1}`,
-          header: () => (
-            <TableHeader
-              smallText={`Variable ${idx + 1}`}
-              text={varName}
-              maxLength={maxTextLength}
-            />
-          ),
-          cell: (info: CellContext<EvalRow, string>) => (
-            <TruncatedText text={info.getValue()} maxLength={maxTextLength} />
-          ),
-          // Minimize the size of Variable columns.
-          size: 50,
-        }),
+        columnHelper.accessor(
+          (row: EvalRow) => {
+            return row.vars[idx];
+          },
+          {
+            id: `Variable ${idx + 1}`,
+            header: () => (
+              <TableHeader
+                smallText={`Variable ${idx + 1}`}
+                text={varName}
+                maxLength={maxTextLength}
+              />
+            ),
+            cell: (info: CellContext<EvalRow, string>) => (
+              <TruncatedText text={info.getValue()} maxLength={maxTextLength} />
+            ),
+            // Minimize the size of Variable columns.
+            size: 50,
+          },
+        ),
       ),
     }),
     columnHelper.group({
@@ -228,11 +276,13 @@ export default function ResultsTable({
               numGood[idx] === highestPassingCount && highestPassingCount !== 0;
             const columnId = `Prompt ${idx + 1}`;
             const isChecked = failureFilter[columnId] || false;
+            // TODO(ian): prompt string support for backwards compatibility, remove after 0.17.0
             return (
               <>
                 <TableHeader
                   smallText={`Prompt ${idx + 1}`}
-                  text={prompt}
+                  text={typeof prompt === 'string' ? prompt : prompt.display}
+                  expandedText={typeof prompt === 'string' ? undefined : prompt.raw}
                   maxLength={maxTextLength}
                 />
                 {filterMode === 'failures' && (
@@ -258,7 +308,7 @@ export default function ResultsTable({
             );
           },
           cell: (info: CellContext<EvalRow, string>) => (
-            <PromptOutput
+            <EvalOutputCell
               output={info.getValue() as unknown as EvalRowOutput}
               maxTextLength={maxTextLength}
               rowIndex={info.row.index}

package/src/web/client/src/types.ts

@@ -1,12 +1,18 @@
+type Prompt = {
+  display: string;
+  raw: string;
+};
+
 export type EvalHead = {
-  prompts: string[];
+  prompts: Prompt[];
   vars: string[];
 };
 
 export type EvalRowOutput = {
   pass: boolean;
   score: number;
-  text: string;
+  text: string | object;
+  prompt: string;
 };
 
 export type EvalRow = {