inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/samples/SampleSummaryView.tsx

@@ -1,17 +1,18 @@
 import clsx from "clsx";
 import { MarkdownDiv } from "../components/MarkdownDiv";
-import { EvalSample, WorkingTime } from "../types/log";
+import { EvalSample, Target, TotalTime, WorkingTime } from "../types/log";
 import { arrayToString, formatTime, inputString } from "../utils/format";
-import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
 import { FlatSampleError } from "./error/FlatSampleErrorView";
 
 import { FC, ReactNode } from "react";
+import { SampleSummary } from "../api/types";
+import { useSampleDescriptor, useScore } from "../state/hooks";
 import styles from "./SampleSummaryView.module.css";
+import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
 
 interface SampleSummaryViewProps {
   parent_id: string;
-  sample: EvalSample;
-  sampleDescriptor: SamplesDescriptor;
+  sample: SampleSummary | EvalSample;
 }
 
 interface SummaryColumn {
@@ -23,14 +24,73 @@ interface SummaryColumn {
   title?: string;
 }
 
+interface SampleFields {
+  id: string | number;
+  input: string[];
+  target: Target;
+  answer?: string;
+  limit?: string;
+  working_time?: WorkingTime;
+  total_time?: TotalTime;
+  error?: string;
+}
+
+function isEvalSample(
+  sample: SampleSummary | EvalSample,
+): sample is EvalSample {
+  return "choices" in sample && Array.isArray((sample as EvalSample).choices);
+}
+
+const resolveSample = (
+  sample: SampleSummary | EvalSample,
+  sampleDescriptor: SamplesDescriptor,
+): SampleFields => {
+  const input = inputString(sample.input);
+  if (isEvalSample(sample) && sample.choices && sample.choices.length > 0) {
+    input.push("");
+    input.push(
+      ...sample.choices.map((choice, index) => {
+        return `${String.fromCharCode(65 + index)}) ${choice}`;
+      }),
+    );
+  }
+
+  const target = sample.target;
+  const answer =
+    sample && sampleDescriptor
+      ? sampleDescriptor.selectedScorerDescriptor(sample)?.answer()
+      : undefined;
+  const limit = isEvalSample(sample) ? sample.limit?.type : undefined;
+  const working_time = isEvalSample(sample) ? sample.working_time : undefined;
+  const total_time = isEvalSample(sample) ? sample.total_time : undefined;
+  const error = isEvalSample(sample) ? sample.error?.message : undefined;
+
+  return {
+    id: sample.id,
+    input,
+    target,
+    answer,
+    limit,
+    working_time,
+    total_time,
+    error,
+  };
+};
+
 /**
  * Component to display a sample with relevant context and visibility control.
  */
 export const SampleSummaryView: FC<SampleSummaryViewProps> = ({
   parent_id,
   sample,
-  sampleDescriptor,
 }) => {
+  const sampleDescriptor = useSampleDescriptor();
+  const currentScore = useScore();
+  if (!sampleDescriptor) {
+    return undefined;
+  }
+  const fields = resolveSample(sample, sampleDescriptor);
+
   const input =
     sampleDescriptor?.messageShape.normalized.input > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.input)
@@ -47,43 +107,33 @@ export const SampleSummaryView: FC<SampleSummaryViewProps> = ({
     sampleDescriptor?.messageShape.normalized.limit > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.limit)
       : 0;
-  const timeSize = sample.working_time || sample.total_time ? 0.15 : 0;
+  const timeSize = fields.working_time || fields.total_time ? 0.15 : 0;
   const idSize = Math.max(
     2,
     Math.min(10, sampleDescriptor?.messageShape.raw.id),
   );
 
-  const scoreInput = inputString(sample.input);
-  if (sample.choices && sample.choices.length > 0) {
-    scoreInput.push("");
-    scoreInput.push(
-      ...sample.choices.map((choice, index) => {
-        return `${String.fromCharCode(65 + index)}) ${choice}`;
-      }),
-    );
-  }
-
   // The columns for the sample
   const columns: SummaryColumn[] = [];
   columns.push({
     label: "Id",
-    value: sample.id,
+    value: fields.id,
     size: `${idSize}em`,
   });
 
   columns.push({
     label: "Input",
-    value: scoreInput,
+    value: <MarkdownDiv markdown={fields.input.join(" ")} />,
     size: `${input}fr`,
     clamp: true,
   });
 
-  if (sample.target) {
+  if (fields.target) {
     columns.push({
       label: "Target",
       value: (
         <MarkdownDiv
-          markdown={arrayToString(arrayToString(sample?.target || "none"))}
+          markdown={arrayToString(fields?.target || "none")}
           className={clsx("no-last-para-padding", styles.target)}
         />
       ),
@@ -92,16 +142,12 @@ export const SampleSummaryView: FC<SampleSummaryViewProps> = ({
     });
   }
 
-  const fullAnswer =
-    sample && sampleDescriptor
-      ? sampleDescriptor.selectedScorerDescriptor(sample).answer()
-      : undefined;
-  if (fullAnswer) {
+  if (fields.answer) {
     columns.push({
       label: "Answer",
       value: sample ? (
         <MarkdownDiv
-          markdown={fullAnswer}
+          markdown={fields.answer}
           className={clsx("no-last-para-padding", styles.answer)}
         />
       ) : (
@@ -119,20 +165,20 @@ export const SampleSummaryView: FC<SampleSummaryViewProps> = ({
     return `Working time: ${formatTime(working_time)}`;
   };
 
-  if (sample.total_time) {
+  if (fields.total_time) {
     columns.push({
       label: "Time",
-      value: formatTime(sample.total_time),
+      value: formatTime(fields.total_time),
       size: `${timeSize}fr`,
       center: true,
-      title: toolTip(sample.working_time),
+      title: toolTip(fields.working_time),
     });
   }
 
-  if (sample?.limit && limitSize > 0) {
+  if (fields?.limit && limitSize > 0) {
     columns.push({
       label: "Limit",
-      value: sample.limit.type,
+      value: fields.limit,
       size: `${limitSize}fr`,
       center: true,
     });
@@ -140,11 +186,11 @@ export const SampleSummaryView: FC<SampleSummaryViewProps> = ({
 
   columns.push({
     label: "Score",
-    value: sample.error ? (
-      <FlatSampleError message={sample.error.message} />
+    value: fields.error ? (
+      <FlatSampleError message={fields.error} />
     ) : (
-      // TODO: Cleanup once the PR lands which makes sample / sample summary share common interface
-      sampleDescriptor?.selectedScore(sample)?.render() || ""
+      sampleDescriptor?.evalDescriptor.score(sample, currentScore)?.render() ||
+      ""
     ),
     size: "minmax(2em, 30em)",
     center: true,
@@ -184,9 +230,10 @@ export const SampleSummaryView: FC<SampleSummaryViewProps> = ({
           <div
             key={`sample-summ-val-${idx}`}
             className={clsx(
+              styles.value,
               styles.wrap,
               col.clamp ? "three-line-clamp" : undefined,
-              col.center ? styles.centerLabel : undefined,
+              col.center ? styles.centerValue : undefined,
             )}
           >
             {col.value}

inspect_ai/_view/www/src/samples/SamplesTools.tsx

@@ -1,47 +1,40 @@
 import { FC } from "react";
 import { Fragment } from "react/jsx-runtime";
-import { ScoreFilter, ScoreLabel } from "../types";
-import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
+import { SampleSummary } from "../api/types";
+import { useScore, useScores } from "../state/hooks";
+import { useStore } from "../state/store";
 import { EpochFilter } from "./sample-tools/EpochFilter";
 import { SampleFilter } from "./sample-tools/sample-filter/SampleFilter";
 import { SelectScorer } from "./sample-tools/SelectScorer";
 import { SortFilter } from "./sample-tools/SortFilter";
 
 interface SampleToolsProps {
-  epoch: string;
-  setEpoch: (epoch: string) => void;
-  epochs: number;
-  scoreFilter: ScoreFilter;
-  setScoreFilter: (filter: ScoreFilter) => void;
-  sort: string;
-  setSort: (sort: string) => void;
-  score?: ScoreLabel;
-  setScore: (score: ScoreLabel) => void;
-  scores: ScoreLabel[];
-  sampleDescriptor: SamplesDescriptor;
+  samples: SampleSummary[];
 }
 
-export const SampleTools: FC<SampleToolsProps> = ({
-  epoch,
-  setEpoch,
-  epochs,
-  scoreFilter,
-  setScoreFilter,
-  sort,
-  setSort,
-  score,
-  setScore,
-  scores,
-  sampleDescriptor,
-}) => {
+export const SampleTools: FC<SampleToolsProps> = ({ samples }) => {
+  const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
+
+  const filter = useStore((state) => state.log.filter);
+  const setFilter = useStore((state) => state.logActions.setFilter);
+
+  const scores = useScores();
+  const score = useScore();
+  const setScore = useStore((state) => state.logActions.setScore);
+  const epoch = useStore((state) => state.log.epoch);
+  const setEpoch = useStore((state) => state.logActions.setEpoch);
+  const sort = useStore((state) => state.log.sort);
+  const setSort = useStore((state) => state.logActions.setSort);
+
+  const epochs = selectedLogSummary?.eval.config.epochs || 1;
   return (
     <Fragment>
       <SampleFilter
-        evalDescriptor={sampleDescriptor.evalDescriptor}
-        scoreFilter={scoreFilter}
-        setScoreFilter={setScoreFilter}
+        samples={samples}
+        scoreFilter={filter}
+        setScoreFilter={setFilter}
       />
-      {scores.length > 1 ? (
+      {scores?.length > 1 ? (
         <SelectScorer scores={scores} score={score} setScore={setScore} />
       ) : undefined}
       {epochs > 1 ? (
@@ -51,3 +44,15 @@ export const SampleTools: FC<SampleToolsProps> = ({
     </Fragment>
   );
 };
+
+interface ScoreFilterToolsProps {}
+
+export const ScoreFilterTools: FC<ScoreFilterToolsProps> = () => {
+  const scores = useScores();
+  const score = useScore();
+  const setScore = useStore((state) => state.logActions.setScore);
+  if (scores.length <= 1) {
+    return undefined;
+  }
+  return <SelectScorer scores={scores} score={score} setScore={setScore} />;
+};

inspect_ai/_view/www/src/samples/chat/ChatMessage.tsx

@@ -46,8 +46,9 @@ export const ChatMessage: FC<ChatMessageProps> = ({
           indented ? styles.indented : undefined,
         )}
       >
-        <ExpandablePanel collapse={collapse} lines={30}>
+        <ExpandablePanel id={`${id}-message`} collapse={collapse} lines={30}>
           <MessageContents
+            id={`${id}-contents`}
             key={`${id}-contents`}
             message={message}
             toolMessages={toolMessages}

inspect_ai/_view/www/src/samples/chat/ChatMessageRenderer.tsx

@@ -26,7 +26,7 @@ export const ChatMessageRenderer: ContentRenderer = {
   render: (id, entry) => {
     return {
       rendered: (
-        <NavPills>
+        <NavPills id={`${id}-navpills`}>
           <ChatSummary title="Last Turn" id={id} messages={entry.value} />
           <ChatView title="All" id={id} messages={entry.value} />
         </NavPills>

inspect_ai/_view/www/src/samples/chat/ChatViewVirtualList.tsx

@@ -1,73 +1,65 @@
-import { FC, RefObject, useState } from "react";
+import { FC, memo, ReactNode, RefObject, useMemo } from "react";
 import { Messages } from "../../types/log";
 
-import clsx from "clsx";
-import { Virtuoso } from "react-virtuoso";
 import { ChatMessageRow } from "./ChatMessageRow";
 import { ResolvedMessage, resolveMessages } from "./messages";
 
-import styles from "./ChatViewVirtualList.module.css";
+import { LiveVirtualList } from "../../components/LiveVirtualList";
 
 interface ChatViewVirtualListProps {
-  id?: string;
+  id: string;
+  className?: string | string[];
   messages: Messages;
   toolCallStyle: "compact" | "complete";
-  className?: string | string[];
   indented: boolean;
   numbered?: boolean;
-  scrollRef?: RefObject<HTMLElement | null>;
+  scrollRef?: RefObject<HTMLDivElement | null>;
+  running?: boolean;
 }
 
 /**
  * Renders the ChatViewVirtualList component.
  */
-export const ChatViewVirtualList: FC<ChatViewVirtualListProps> = ({
-  id,
-  messages,
-  toolCallStyle,
-  className,
-  indented,
-  numbered = true,
-  scrollRef,
-}) => {
-  const collapsedMessages = resolveMessages(messages);
-  const [followOutput, setFollowOutput] = useState(false);
+export const ChatViewVirtualList: FC<ChatViewVirtualListProps> = memo(
+  ({
+    id,
+    messages,
+    className,
+    toolCallStyle,
+    indented,
+    numbered = true,
+    scrollRef,
+    running,
+  }) => {
+    const collapsedMessages = useMemo(() => {
+      return resolveMessages(messages);
+    }, [messages]);
+
+    const renderRow = (index: number, item: ResolvedMessage): ReactNode => {
+      const number =
+        collapsedMessages.length > 1 && numbered ? index + 1 : undefined;
+
+      return (
+        <ChatMessageRow
+          parentName={id || "chat-virtual-list"}
+          number={number}
+          resolvedMessage={item}
+          indented={indented}
+          toolCallStyle={toolCallStyle}
+        />
+      );
+    };
 
-  const renderRow = (item: ResolvedMessage, index: number) => {
-    const number =
-      collapsedMessages.length > 1 && numbered ? index + 1 : undefined;
     return (
-      <ChatMessageRow
-        parentName={id || "chat-virtual-list"}
-        number={number}
-        resolvedMessage={item}
-        indented={indented}
-        toolCallStyle={toolCallStyle}
+      <LiveVirtualList<ResolvedMessage>
+        id="chat-virtual-list"
+        className={className}
+        scrollRef={scrollRef}
+        data={collapsedMessages}
+        renderRow={renderRow}
+        live={running}
+        showProgress={running}
       />
     );
-  };
-
-  const result = (
-    <Virtuoso
-      customScrollParent={scrollRef?.current ? scrollRef.current : undefined}
-      style={{ height: "100%", width: "100%" }}
-      data={collapsedMessages}
-      itemContent={(index: number, data: ResolvedMessage) => {
-        return renderRow(data, index);
-      }}
-      increaseViewportBy={{ top: 1000, bottom: 1000 }}
-      overscan={{
-        main: 10,
-        reverse: 10,
-      }}
-      followOutput={followOutput}
-      atBottomStateChange={(atBottom: boolean) => {
-        setFollowOutput(atBottom);
-      }}
-      skipAnimationFrameInResizeObserver={true}
-      className={clsx(styles.list, className)}
-    />
-  );
-
-  return result;
-};
+  },
+);

inspect_ai/_view/www/src/samples/chat/MessageContent.tsx

@@ -52,6 +52,7 @@ export const MessageContent: FC<MessageContentProps> = ({ contents }) => {
           {
             type: "text",
             text: content,
+            refusal: null,
           },
           index === contents.length - 1,
         );
@@ -75,6 +76,7 @@ export const MessageContent: FC<MessageContentProps> = ({ contents }) => {
     const contentText: ContentText = {
       type: "text",
       text: contents,
+      refusal: null,
     };
     return messageRenderers["text"].render(
       "text-message-content",
@@ -104,6 +106,9 @@ const messageRenderers: Record<string, MessageRenderer> = {
   reasoning: {
     render: (key, content, isLast) => {
       const r = content as ContentReasoning;
+      if (!r.reasoning && !r.redacted) {
+        return undefined;
+      }
       return (
         <Fragment key={key}>
           <div
@@ -115,7 +120,7 @@ const messageRenderers: Record<string, MessageRenderer> = {
           >
             Reasoning
           </div>
-          <ExpandablePanel collapse={true}>
+          <ExpandablePanel id={`${key}-reasoning`} collapse={true}>
             <MarkdownDiv
               markdown={
                 r.redacted

inspect_ai/_view/www/src/samples/chat/MessageContents.tsx

@@ -13,12 +13,14 @@ import { ContentTool } from "../../types";
 import styles from "./MessageContents.module.css";
 
 interface MessageContentsProps {
+  id: string;
   message: ChatMessageAssistant | ChatMessageSystem | ChatMessageUser;
   toolMessages: ChatMessageTool[];
   toolCallStyle: "compact" | "complete";
 }
 
 export const MessageContents: FC<MessageContentsProps> = ({
+  id,
   message,
   toolMessages,
   toolCallStyle,
@@ -56,6 +58,7 @@ export const MessageContents: FC<MessageContentsProps> = ({
       } else {
         return (
           <ToolCallView
+            id={`${id}-tool-call`}
             key={`tool-call-${idx}`}
             functionCall={functionCall}
             input={input}
@@ -98,6 +101,7 @@ const resolveToolMessage = (toolMessage?: ChatMessageTool): ContentTool[] => {
           {
             type: "text",
             text: content,
+            refusal: null,
           },
         ],
       },
@@ -112,6 +116,7 @@ const resolveToolMessage = (toolMessage?: ChatMessageTool): ContentTool[] => {
               {
                 type: "text",
                 text: con,
+                refusal: null,
               },
             ],
           } as ContentTool;

inspect_ai/_view/www/src/samples/chat/messages.ts

@@ -9,6 +9,7 @@ import {
   ContentReasoning,
   ContentText,
   ContentVideo,
+  Events,
   Messages,
 } from "../../types/log";
 
@@ -65,9 +66,11 @@ export const resolveMessages = (messages: Messages) => {
   }
 
   const systemMessage: ChatMessageSystem = {
+    id: "sys-message-6815A84B062A",
     role: "system",
     content: systemContent,
     source: "input",
+    internal: null,
   };
 
   // Converge them
@@ -118,8 +121,41 @@ const normalizeContent = (
     return {
       type: "text",
       text: content,
+      refusal: null,
     };
   } else {
     return content;
   }
 };
+
+export const messagesFromEvents = (runningEvents: Events): Messages => {
+  const messages: Map<
+    string,
+    ChatMessageSystem | ChatMessageUser | ChatMessageAssistant | ChatMessageTool
+  > = new Map();
+
+  runningEvents
+    .filter((e) => e.event === "model")
+    .forEach((e) => {
+      for (const m of e.input) {
+        const inputMessage = m as
+          | ChatMessageSystem
+          | ChatMessageUser
+          | ChatMessageAssistant
+          | ChatMessageTool;
+        if (inputMessage.id && !messages.has(inputMessage.id)) {
+          messages.set(inputMessage.id, inputMessage);
+        }
+      }
+      const outputMessage = e.output.choices[0].message;
+      if (outputMessage.id) {
+        messages.set(outputMessage.id, outputMessage);
+      }
+    });
+
+  if (messages.size > 0) {
+    return messages.values().toArray();
+  } else {
+    return [];
+  }
+};

inspect_ai/_view/www/src/samples/chat/tools/ToolCallView.module.css

@@ -0,0 +1,3 @@
+.output {
+  padding-top: 1em;
+}

inspect_ai/_view/www/src/samples/chat/tools/ToolCallView.tsx

@@ -10,10 +10,12 @@ import {
   ToolCallContent,
 } from "../../../types/log";
 import { MessageContent } from "../MessageContent";
+import styles from "./ToolCallView.module.css";
 import { ToolInput } from "./ToolInput";
 import { ToolTitle } from "./ToolTitle";
 
 interface ToolCallViewProps {
+  id: string;
   functionCall: string;
   input?: string;
   highlightLanguage?: string;
@@ -43,6 +45,7 @@ interface ToolCallViewProps {
  * Renders the ToolCallView component.
  */
 export const ToolCallView: FC<ToolCallViewProps> = ({
+  id,
   functionCall,
   input,
   highlightLanguage,
@@ -116,7 +119,13 @@ export const ToolCallView: FC<ToolCallViewProps> = ({
             toolCallView={view}
           />
           {hasContent ? (
-            <ExpandablePanel collapse={collapse} border={true} lines={15}>
+            <ExpandablePanel
+              id={`${id}-tool-input`}
+              collapse={collapse}
+              border={true}
+              lines={15}
+              className={styles.output}
+            >
               <MessageContent contents={normalizedContent} />
             </ExpandablePanel>
           ) : undefined}
@@ -166,6 +175,7 @@ const normalizeContent = (
           {
             type: "text",
             text: String(output),
+            refusal: null,
           },
         ],
       },