inspect-ai 0.3.80__py3-none-any.whl → 0.3.82__py3-none-any.whl

inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.tsx

@@ -1,7 +1,7 @@
 import clsx from "clsx";
 import { ScoreLabel } from "../../types";
 
-import { FC } from "react";
+import { ChangeEvent, FC, useCallback } from "react";
 import styles from "./SelectScorer.module.css";
 
 interface SelectScorerProps {
@@ -26,6 +26,13 @@ export const SelectScorer: FC<SelectScorerProps> = ({
     return accum;
   }, [] as ScoreLabel[]);
 
+  const handleSelectScore = useCallback(
+    (index: number) => {
+      setScore(scores[index]);
+    },
+    [setScore, scores],
+  );
+
   if (scorers.length === 1) {
     // There is only a single scorer in play, just show the list of available scores
     return (
@@ -44,9 +51,7 @@ export const SelectScorer: FC<SelectScorerProps> = ({
         <ScoreSelector
           scores={scores}
           selectedIndex={scoreIndex(scores, score)}
-          setSelectedIndex={(index: number) => {
-            setScore(scores[index]);
-          }}
+          setSelectedIndex={handleSelectScore}
         />
       </div>
     );
@@ -75,18 +80,14 @@ export const SelectScorer: FC<SelectScorerProps> = ({
         <ScorerSelector
           scorers={scorers}
           selectedIndex={scorerIndex(scorers, score)}
-          setSelectedIndex={(index: number) => {
-            setScore(scorers[index]);
-          }}
+          setSelectedIndex={handleSelectScore}
         />
         {scorerScores.length > 1 ? (
           <ScoreSelector
             className={clsx(styles.secondSel)}
             scores={scorerScores}
             selectedIndex={scoreIndex(scorerScores, score)}
-            setSelectedIndex={(index: number) => {
-              setScore(scorerScores[index]);
-            }}
+            setSelectedIndex={handleSelectScore}
           />
         ) : undefined}
       </div>
@@ -107,6 +108,14 @@ const ScoreSelector: FC<ScoreSelectorProps> = ({
   setSelectedIndex,
   className,
 }) => {
+  const handleChange = useCallback(
+    (e: ChangeEvent<HTMLSelectElement>) => {
+      const sel = e.target as HTMLSelectElement;
+      setSelectedIndex(sel.selectedIndex);
+    },
+    [setSelectedIndex],
+  );
+
   return (
     <select
       className={clsx(
@@ -117,10 +126,7 @@ const ScoreSelector: FC<ScoreSelectorProps> = ({
       )}
       aria-label=".select-scorer-label"
       value={scores[selectedIndex].name}
-      onChange={(e) => {
-        const sel = e.target as HTMLSelectElement;
-        setSelectedIndex(sel.selectedIndex);
-      }}
+      onChange={handleChange}
     >
       {scores.map((score) => {
         return (
@@ -144,15 +150,20 @@ const ScorerSelector: FC<ScorerSelectorProps> = ({
   selectedIndex,
   setSelectedIndex,
 }) => {
+  const handleChange = useCallback(
+    (e: ChangeEvent<HTMLSelectElement>) => {
+      const sel = e.target as HTMLSelectElement;
+      setSelectedIndex(sel.selectedIndex);
+    },
+    [setSelectedIndex],
+  );
+
   return (
     <select
       className={clsx("form-select", "form-select-sm", "text-size-smaller")}
       aria-label=".epoch-filter-label"
       value={scorers[selectedIndex].scorer}
-      onChange={(e) => {
-        const sel = e.target as HTMLSelectElement;
-        setSelectedIndex(sel.selectedIndex);
-      }}
+      onChange={handleChange}
     >
       {scorers.map((scorer) => {
         return (

inspect_ai/_view/www/src/samples/sample-tools/SortFilter.tsx

@@ -1,5 +1,5 @@
 import clsx from "clsx";
-import { FC } from "react";
+import { ChangeEvent, FC, useCallback } from "react";
 import { SampleSummary } from "../../api/types";
 import {
   kEpochAscVal,
@@ -9,6 +9,7 @@ import {
   kScoreAscVal,
   kScoreDescVal,
 } from "../../constants";
+import { ScoreLabel } from "../../types";
 import { isNumeric } from "../../utils/type";
 import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
 import styles from "./SortFilter.module.css";
@@ -42,6 +43,15 @@ export const SortFilter: FC<SortFilterProps> = ({ sort, setSort, epochs }) => {
     label: "score desc",
     val: kScoreDescVal,
   });
+
+  const handleChange = useCallback(
+    (e: ChangeEvent<HTMLSelectElement>) => {
+      const sel = e.target as HTMLSelectElement;
+      setSort(sel.value);
+    },
+    [setSort],
+  );
+
   return (
     <div className={styles.flex}>
       <span
@@ -59,10 +69,7 @@ export const SortFilter: FC<SortFilterProps> = ({ sort, setSort, epochs }) => {
         className={clsx("form-select", "form-select-sm", "text-size-smaller")}
         aria-label=".sort-filter-label"
         value={sort}
-        onChange={(e) => {
-          const sel = e.target as HTMLSelectElement;
-          setSort(sel.value);
-        }}
+        onChange={handleChange}
       >
         {options.map((option) => {
           return (
@@ -102,8 +109,13 @@ export const sortSamples = (
   sort: string,
   samples: SampleSummary[],
   samplesDescriptor: SamplesDescriptor,
-): { sorted: SampleSummary[]; order: "asc" | "desc" } => {
+  score?: ScoreLabel,
+): SampleSummary[] => {
   const sortedSamples = samples.sort((a: SampleSummary, b: SampleSummary) => {
+    const scoreDescriptor = score
+      ? samplesDescriptor.evalDescriptor.scoreDescriptor(score)
+      : undefined;
+
     switch (sort) {
       case kSampleAscVal: {
         const result = sortId(a, b);
@@ -134,50 +146,38 @@ export const sortSamples = (
         if (result !== 0) {
           return result;
         } else {
-          return sortId(a, b);
+          return sortId(b, a);
         }
       }
 
       case kScoreAscVal: {
-        const aScore = samplesDescriptor.selectedScore(a);
-        const bScore = samplesDescriptor.selectedScore(b);
+        const aScore = samplesDescriptor.evalDescriptor.score(a, score);
+        const bScore = samplesDescriptor.evalDescriptor.score(b, score);
         if (
           aScore === undefined ||
           bScore === undefined ||
-          samplesDescriptor.selectedScoreDescriptor == undefined
+          scoreDescriptor === undefined
         ) {
           return 0;
         }
-        return samplesDescriptor.selectedScoreDescriptor.compare(
-          aScore,
-          bScore,
-        );
+        return scoreDescriptor?.compare(aScore, bScore);
       }
       case kScoreDescVal: {
-        const aScore = samplesDescriptor.selectedScore(a);
-        const bScore = samplesDescriptor.selectedScore(b);
+        const aScore = samplesDescriptor.evalDescriptor.score(a, score);
+        const bScore = samplesDescriptor.evalDescriptor.score(b, score);
         if (
           aScore === undefined ||
           bScore === undefined ||
-          samplesDescriptor.selectedScoreDescriptor == undefined
+          scoreDescriptor == undefined
         ) {
           return 0;
         }
 
-        return samplesDescriptor.selectedScoreDescriptor.compare(
-          bScore,
-          aScore,
-        );
+        return scoreDescriptor?.compare(bScore, aScore);
       }
       default:
         return 0;
     }
   });
-  return {
-    sorted: sortedSamples,
-    order:
-      sort === kSampleAscVal || sort === kEpochAscVal || sort === kScoreAscVal
-        ? "asc"
-        : "desc",
-  };
+  return sortedSamples;
 };

inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.tsx

@@ -16,6 +16,8 @@ import clsx from "clsx";
 import { EditorView, minimalSetup } from "codemirror";
 import { FC, useEffect, useMemo, useRef, useState } from "react";
 
+import { SampleSummary } from "../../../api/types";
+import { useEvalDescriptor } from "../../../state/hooks";
 import { ScoreFilter } from "../../../types";
 import { EvalDescriptor } from "../../descriptor/types";
 import { FilterError, filterSamples, scoreFilterItems } from "../filters";
@@ -30,7 +32,7 @@ interface FilteringResult {
 }
 
 interface SampleFilterProps {
-  evalDescriptor: EvalDescriptor;
+  samples: SampleSummary[];
   scoreFilter: ScoreFilter;
   setScoreFilter: (filter: ScoreFilter) => void;
 }
@@ -105,11 +107,12 @@ const editorTheme = EditorView.theme({
 // Helper functions
 const getFilteringResult = (
   evalDescriptor: EvalDescriptor,
+  sampleSummaries: SampleSummary[],
   filterValue: string,
 ): FilteringResult => {
   const { result, error } = filterSamples(
     evalDescriptor,
-    evalDescriptor.samples,
+    sampleSummaries,
     filterValue,
   );
   return { numSamples: result.length, error };
@@ -151,7 +154,7 @@ const getLints = (
 
 // Main component
 export const SampleFilter: FC<SampleFilterProps> = ({
-  evalDescriptor,
+  samples,
   scoreFilter,
   setScoreFilter,
 }) => {
@@ -160,9 +163,10 @@ export const SampleFilter: FC<SampleFilterProps> = ({
   const linterCompartment = useRef<Compartment>(new Compartment());
   const autocompletionCompartment = useRef<Compartment>(new Compartment());
   const updateListenerCompartment = useRef<Compartment>(new Compartment());
+  const evalDescriptor = useEvalDescriptor();
 
   const filterItems = useMemo(
-    () => scoreFilterItems(evalDescriptor),
+    () => (evalDescriptor ? scoreFilterItems(evalDescriptor) : []),
     [evalDescriptor],
   );
 
@@ -186,9 +190,13 @@ export const SampleFilter: FC<SampleFilterProps> = ({
 
   const makeUpdateListener = () =>
     EditorView.updateListener.of((update) => {
-      if (update.docChanged) {
+      if (update.docChanged && evalDescriptor) {
         const newValue = update.state.doc.toString();
-        const filteringResult = getFilteringResult(evalDescriptor, newValue);
+        const filteringResult = getFilteringResult(
+          evalDescriptor,
+          samples,
+          newValue,
+        );
         if (!filteringResult.error) {
           setScoreFilter({ value: newValue });
         }
@@ -229,9 +237,11 @@ export const SampleFilter: FC<SampleFilterProps> = ({
     const currentValue = editorViewRef.current.state.doc.toString();
     if (scoreFilter.value === currentValue) return;
 
-    setFilteringResultInstant(
-      getFilteringResult(evalDescriptor, scoreFilter.value || ""),
-    );
+    if (evalDescriptor) {
+      setFilteringResultInstant(
+        getFilteringResult(evalDescriptor, samples, scoreFilter.value || ""),
+      );
+    }
     editorViewRef.current.dispatch({
       changes: {
         from: 0,

inspect_ai/_view/www/src/samples/sampleDataAdapter.ts

@@ -0,0 +1,33 @@
+import { EventData, SampleData } from "../api/types";
+import { Events } from "../types/log";
+import { resolveAttachments } from "../utils/attachments";
+
+export const sampleDataAdapter = () => {
+  const attachments: Record<string, string> = {};
+  const events: Record<string, EventData> = {};
+
+  return {
+    addData: (data: SampleData) => {
+      data.attachments.forEach((a) => {
+        if (attachments[a.hash] === undefined) {
+          attachments[a.hash] = a.content;
+        }
+      });
+
+      data.events.forEach((e) => {
+        if (events[e.event_id] === undefined) {
+          events[e.event_id] = e;
+        }
+      });
+    },
+    resolvedEvents: (): Events => {
+      const eventDatas = Object.values(events);
+
+      const resolvedEvents = eventDatas.map((ed: EventData) => {
+        return ed.event;
+      }) as Events;
+
+      return resolveAttachments<Events>(resolvedEvents, attachments);
+    },
+  };
+};

inspect_ai/_view/www/src/samples/sampleLimit.ts

@@ -1,9 +1,9 @@
-import { Type11 } from "../types/log";
+import { Type14 } from "../types/log";
 
 /**
  * Formats a limit message
  */
-export const sampleLimitMessage = (type: Type11): string => {
+export const sampleLimitMessage = (type: Type14): string => {
   switch (type) {
     case "operator":
       return "Sample terminated due to operator limit.";

inspect_ai/_view/www/src/samples/scores/SampleScoreView.tsx

@@ -4,27 +4,26 @@ import { MarkdownDiv } from "../../components/MarkdownDiv";
 import { MetaDataGrid } from "../../metadata/MetaDataGrid";
 import { EvalSample } from "../../types/log";
 import { arrayToString, inputString } from "../../utils/format";
-import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
 import { SampleScores } from "./SampleScores";
 
 import { FC } from "react";
 import { SampleSummary } from "../../api/types";
+import { useEvalDescriptor } from "../../state/hooks";
 import styles from "./SampleScoreView.module.css";
 
 interface SampleScoreViewProps {
   sample: EvalSample;
-  sampleDescriptor: SamplesDescriptor;
   scorer: string;
   className?: string | string[];
 }
 
 export const SampleScoreView: FC<SampleScoreViewProps> = ({
   sample,
-  sampleDescriptor,
   className,
   scorer,
 }) => {
-  if (!sampleDescriptor) {
+  const evalDescriptor = useEvalDescriptor();
+  if (!evalDescriptor) {
     return null;
   }
 
@@ -38,10 +37,10 @@ export const SampleScoreView: FC<SampleScoreViewProps> = ({
     );
   }
 
-  const scorerDescriptor = sampleDescriptor.evalDescriptor.scorerDescriptor(
-    sample,
-    { scorer, name: scorer },
-  );
+  const scorerDescriptor = evalDescriptor.scorerDescriptor(sample, {
+    scorer,
+    name: scorer,
+  });
   const explanation = scorerDescriptor.explanation() || "(No Explanation)";
   const answer = scorerDescriptor.answer();
   const metadata = scorerDescriptor.metadata();
@@ -128,7 +127,6 @@ export const SampleScoreView: FC<SampleScoreViewProps> = ({
                 <td className={clsx(styles.scoreValue)}>
                   <SampleScores
                     sample={sample as any as SampleSummary}
-                    sampleDescriptor={sampleDescriptor}
                     scorer={scorer}
                   />
                 </td>

inspect_ai/_view/www/src/samples/scores/SampleScores.tsx

@@ -1,30 +1,26 @@
 import { FC, Fragment } from "react";
 import { SampleSummary } from "../../api/types";
-import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
 
+import { useSampleDescriptor } from "../../state/hooks";
 import styles from "./SampleScores.module.css";
 
 interface SampleScoresProps {
   sample: SampleSummary;
-  sampleDescriptor: SamplesDescriptor;
   scorer: string;
 }
 
-export const SampleScores: FC<SampleScoresProps> = ({
-  sample,
-  sampleDescriptor,
-  scorer,
-}) => {
+export const SampleScores: FC<SampleScoresProps> = ({ sample, scorer }) => {
+  const samplesDescriptor = useSampleDescriptor();
   const scores = scorer
-    ? sampleDescriptor.evalDescriptor
+    ? samplesDescriptor?.evalDescriptor
         .scorerDescriptor(sample, { scorer, name: scorer })
         .scores()
-    : sampleDescriptor.selectedScorerDescriptor(sample).scores();
+    : samplesDescriptor?.selectedScorerDescriptor(sample)?.scores();
 
-  if (scores.length === 1) {
+  if (scores?.length === 1) {
     return scores[0].rendered();
   } else {
-    const rows = scores.map((score) => {
+    const rows = scores?.map((score) => {
       return (
         <Fragment>
           <div style={{ opacity: "0.7" }}>{score.name}</div>

inspect_ai/_view/www/src/samples/transcript/ErrorEventView.tsx

@@ -4,13 +4,10 @@ import { ANSIDisplay } from "../../components/AnsiDisplay";
 import { ErrorEvent } from "../../types/log";
 import { formatDateTime } from "../../utils/format";
 import { EventPanel } from "./event/EventPanel";
-import { TranscriptEventState } from "./types";
 
 interface ErrorEventViewProps {
   id: string;
   event: ErrorEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   className?: string | string[];
 }
 
@@ -20,8 +17,6 @@ interface ErrorEventViewProps {
 export const ErrorEventView: FC<ErrorEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   className,
 }) => {
   return (
@@ -31,14 +26,6 @@ export const ErrorEventView: FC<ErrorEventViewProps> = ({
       className={className}
       subTitle={formatDateTime(new Date(event.timestamp))}
       icon={ApplicationIcons.error}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav: string) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed: boolean) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       <ANSIDisplay
         output={event.error.traceback_ansi}

inspect_ai/_view/www/src/samples/transcript/InfoEventView.tsx

@@ -6,13 +6,10 @@ import { InfoEvent } from "../../types/log";
 import { formatDateTime } from "../../utils/format";
 import { EventPanel } from "./event/EventPanel";
 import styles from "./InfoEventView.module.css";
-import { TranscriptEventState } from "./types";
 
 interface InfoEventViewProps {
   id: string;
   event: InfoEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   className?: string | string[];
 }
 
@@ -22,8 +19,6 @@ interface InfoEventViewProps {
 export const InfoEventView: FC<InfoEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   className,
 }) => {
   const panels = [];
@@ -40,14 +35,6 @@ export const InfoEventView: FC<InfoEventViewProps> = ({
       className={className}
       subTitle={formatDateTime(new Date(event.timestamp))}
       icon={ApplicationIcons.info}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       {panels}
     </EventPanel>

inspect_ai/_view/www/src/samples/transcript/InputEventView.tsx

@@ -4,13 +4,10 @@ import { ANSIDisplay } from "../../components/AnsiDisplay";
 import { InputEvent } from "../../types/log";
 import { formatDateTime } from "../../utils/format";
 import { EventPanel } from "./event/EventPanel";
-import { TranscriptEventState } from "./types";
 
 interface InputEventViewProps {
   id: string;
   event: InputEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   className?: string | string[];
 }
 
@@ -20,8 +17,6 @@ interface InputEventViewProps {
 export const InputEventView: FC<InputEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   className,
 }) => {
   return (
@@ -31,14 +26,6 @@ export const InputEventView: FC<InputEventViewProps> = ({
       className={className}
       subTitle={formatDateTime(new Date(event.timestamp))}
       icon={ApplicationIcons.input}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       <ANSIDisplay
         output={event.input_ansi}

inspect_ai/_view/www/src/samples/transcript/ModelEventView.module.css

@@ -41,3 +41,7 @@
   column-gap: 1em;
   row-gap: 0.5em;
 }
+
+.progress {
+  margin-left: 2em;
+}

inspect_ai/_view/www/src/samples/transcript/ModelEventView.tsx

@@ -3,7 +3,7 @@ import "prismjs/components/prism-json";
 import "prismjs/components/prism-python";
 
 import clsx from "clsx";
-import { FC, Fragment, useEffect, useMemo, useRef } from "react";
+import { FC, Fragment, useMemo } from "react";
 import { ApplicationIcons } from "../../appearance/icons";
 import { MetaDataGrid } from "../../metadata/MetaDataGrid";
 import {
@@ -17,9 +17,9 @@ import { ModelUsagePanel } from "../../usage/ModelUsagePanel";
 import { ChatView } from "../chat/ChatView";
 import { EventPanel } from "./event/EventPanel";
 import { EventSection } from "./event/EventSection";
-import { TranscriptEventState } from "./types";
 
-import { highlightElement } from "prismjs";
+import { PulsingDots } from "../../components/PulsingDots";
+import { usePrismHighlight } from "../../state/hooks";
 import styles from "./ModelEventView.module.css";
 import { EventTimingPanel } from "./event/EventTimingPanel";
 import { formatTiming, formatTitle } from "./event/utils";
@@ -27,8 +27,6 @@ import { formatTiming, formatTitle } from "./event/utils";
 interface ModelEventViewProps {
   id: string;
   event: ModelEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   className?: string | string[];
 }
 
@@ -38,8 +36,6 @@ interface ModelEventViewProps {
 export const ModelEventView: FC<ModelEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   className,
 }) => {
   const totalUsage = event.output.usage?.total_tokens;
@@ -74,14 +70,6 @@ export const ModelEventView: FC<ModelEventViewProps> = ({
       title={formatTitle(`Model Call: ${event.model}`, totalUsage, callTime)}
       subTitle={formatTiming(event.timestamp, event.working_start)}
       icon={ApplicationIcons.model}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       <div data-name="Summary" className={styles.container}>
         <ChatView
@@ -91,6 +79,11 @@ export const ModelEventView: FC<ModelEventViewProps> = ({
           numbered={false}
           toolCallStyle="compact"
         />
+        {event.pending ? (
+          <div className={clsx(styles.progress)}>
+            <PulsingDots subtle={false} size="medium" />
+          </div>
+        ) : undefined}
       </div>
       <div data-name="All" className={styles.container}>
         <div className={styles.all}>
@@ -170,27 +163,20 @@ interface APICodeCellProps {
 }
 
 export const APICodeCell: FC<APICodeCellProps> = ({ id, contents }) => {
-  const codeRef = useRef<HTMLElement>(null);
   const sourceCode = useMemo(() => {
     return JSON.stringify(contents, undefined, 2);
   }, [contents]);
-
-  useEffect(() => {
-    if (codeRef.current) {
-      highlightElement(codeRef.current);
-    }
-  }, [contents]);
+  const prismParentRef = usePrismHighlight(sourceCode);
 
   if (!contents) {
     return null;
   }
 
   return (
-    <div className={clsx("model-call")}>
+    <div ref={prismParentRef} className={clsx("model-call")}>
       <pre className={clsx(styles.codePre)}>
         <code
           id={id}
-          ref={codeRef}
           className={clsx("language-json", styles.code, "text-size-small")}
         >
           {sourceCode}