inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/samples/list/SampleList.tsx

@@ -1,15 +1,14 @@
 import {
   FC,
   KeyboardEvent,
+  memo,
   RefObject,
   useCallback,
   useEffect,
   useMemo,
   useRef,
-  useState,
 } from "react";
 import { Virtuoso, VirtuosoHandle } from "react-virtuoso";
-import { EmptyPanel } from "../../components/EmptyPanel";
 import { MessageBand } from "../../components/MessageBand";
 import { formatNoDecimal } from "../../utils/format";
 import { ListItem } from "../../workspace/tabs/types";
@@ -18,6 +17,9 @@ import { SampleRow } from "./SampleRow";
 import { SampleSeparator } from "./SampleSeparator";
 
 import clsx from "clsx";
+import { useProperty, useSampleDescriptor } from "../../state/hooks";
+import { useVirtuosoState } from "../../state/scrolling";
+import { useStore } from "../../state/store";
 import { SampleFooter } from "./SampleFooter";
 import { SampleHeader } from "./SampleHeader";
 import styles from "./SampleList.module.css";
@@ -27,8 +29,8 @@ const kSeparatorHeight = 24;
 
 interface SampleListProps {
   items: ListItem[];
-  sampleDescriptor: SamplesDescriptor;
-  selectedIndex: number;
+  totalItemCount: number;
+  running: boolean;
   nextSample: () => void;
   prevSample: () => void;
   showSample: (index: number) => void;
@@ -36,11 +38,13 @@ interface SampleListProps {
   listHandle: RefObject<VirtuosoHandle | null>;
 }
 
-export const SampleList: FC<SampleListProps> = (props) => {
+export const kSampleFollowProp = "sample-list";
+
+export const SampleList: FC<SampleListProps> = memo((props) => {
   const {
     items,
-    sampleDescriptor,
-    selectedIndex,
+    totalItemCount,
+    running,
     nextSample,
     prevSample,
     showSample,
@@ -48,37 +52,56 @@ export const SampleList: FC<SampleListProps> = (props) => {
     listHandle,
   } = props;
 
-  const [followOutput, setFollowOutput] = useState(false);
+  const { getRestoreState, isScrolling } = useVirtuosoState(
+    listHandle,
+    "sample-list",
+  );
 
-  const [hidden, setHidden] = useState(false);
-  useEffect(() => {
-    setHidden(false);
-  }, [items]);
+  const selectedSampleIndex = useStore(
+    (state) => state.log.selectedSampleIndex,
+  );
+  const samplesDescriptor = useSampleDescriptor();
+  const [followOutput, setFollowOutput] = useProperty(
+    kSampleFollowProp,
+    "follow",
+    {
+      defaultValue: !!running,
+    },
+  );
 
-  // Keep a mapping of the indexes to items (skipping separators)
-  const itemRowMapping = useMemo(() => {
-    const rowIndexes: number[] = [];
-    items.forEach((item, index) => {
-      if (item.type === "sample") {
-        rowIndexes.push(index);
-      }
-    });
-    return rowIndexes;
-  }, [items]);
+  // Track whether we were previously running so we can
+  // decide whether to pop up to the top
+  const prevRunningRef = useRef(running);
 
-  const prevSelectedIndexRef = useRef<number>(null);
   useEffect(() => {
-    const listEl = listHandle.current;
-    if (listEl) {
-      requestAnimationFrame(() => {
-        setTimeout(() => {
-          const actualRowIndex = itemRowMapping[selectedIndex];
-          listEl.scrollToIndex(actualRowIndex);
-          prevSelectedIndexRef.current = actualRowIndex;
-        }, 10);
-      });
+    // When we finish running, if we are following output
+    // then scroll up to the top
+    if (
+      !running &&
+      prevRunningRef.current &&
+      followOutput &&
+      listHandle.current
+    ) {
+      setFollowOutput(false);
+      setTimeout(() => {
+        if (listHandle.current) {
+          listHandle.current.scrollTo({ top: 0, behavior: "instant" });
+        }
+      }, 100);
     }
-  }, [selectedIndex, listHandle, itemRowMapping]);
+    prevRunningRef.current = running;
+  }, [running, followOutput, listHandle]);
+
+  const loaded = useRef(false);
+  const handleAtBottomStateChange = useCallback(
+    (atBottom: boolean) => {
+      if (loaded.current && running) {
+        setFollowOutput(atBottom);
+      }
+      loaded.current = true;
+    },
+    [running, setFollowOutput, followOutput],
+  );
 
   const onkeydown = useCallback(
     (e: KeyboardEvent<HTMLDivElement>) => {
@@ -94,48 +117,51 @@ export const SampleList: FC<SampleListProps> = (props) => {
           e.stopPropagation();
           break;
         case "Enter":
-          showSample(selectedIndex);
+          showSample(selectedSampleIndex);
           e.preventDefault();
           e.stopPropagation();
           break;
       }
     },
-    [selectedIndex, nextSample, prevSample, showSample],
+    [selectedSampleIndex, nextSample, prevSample, showSample],
   );
 
-  // If there are no samples, just display an empty state
-  if (items.length === 0) {
-    return <EmptyPanel>No Samples</EmptyPanel>;
-  }
+  const gridColumnsTemplate = useMemo(() => {
+    return gridColumnsValue(samplesDescriptor);
+  }, [samplesDescriptor]);
 
-  const renderRow = (item: ListItem) => {
-    if (item.type === "sample") {
-      return (
-        <SampleRow
-          id={`${item.number}`}
-          index={item.index}
-          sample={item.data}
-          height={kSampleHeight}
-          sampleDescriptor={sampleDescriptor}
-          gridColumnsTemplate={gridColumnsValue(sampleDescriptor)}
-          selected={selectedIndex === item.index}
-          showSample={showSample}
-        />
-      );
-    } else if (item.type === "separator") {
-      return (
-        <SampleSeparator
-          id={`sample-group${item.number}`}
-          title={item.data}
-          height={kSeparatorHeight}
-        />
-      );
-    } else {
-      return null;
-    }
-  };
+  const renderRow = useCallback(
+    (_index: number, item: ListItem) => {
+      if (item.type === "sample") {
+        return (
+          <SampleRow
+            id={`${item.number}`}
+            index={item.index}
+            sample={item.data}
+            height={kSampleHeight}
+            answer={item.answer}
+            completed={item.completed}
+            scoreRendered={item.scoreRendered}
+            gridColumnsTemplate={gridColumnsTemplate}
+            showSample={showSample}
+          />
+        );
+      } else if (item.type === "separator") {
+        return (
+          <SampleSeparator
+            id={`sample-group${item.number}`}
+            title={item.data}
+            height={kSeparatorHeight}
+          />
+        );
+      } else {
+        return null;
+      }
+    },
+    [showSample, gridColumnsTemplate],
+  );
 
-  const { input, limit, answer, target } = gridColumns(sampleDescriptor);
+  const { input, limit, answer, target } = gridColumns(samplesDescriptor);
 
   const sampleCount = items?.reduce((prev, current) => {
     if (current.type === "sample") {
@@ -176,68 +202,79 @@ export const SampleList: FC<SampleListProps> = (props) => {
     <div className={styles.mainLayout}>
       {warningMessage ? (
         <MessageBand
+          id={"sample-warning-message"}
           message={warningMessage}
-          hidden={hidden}
-          setHidden={setHidden}
           type="info"
         />
       ) : undefined}
-
       <SampleHeader
         input={input !== "0"}
         target={target !== "0"}
         answer={answer !== "0"}
         limit={limit !== "0"}
-        gridColumnsTemplate={gridColumnsValue(sampleDescriptor)}
+        gridColumnsTemplate={gridColumnsTemplate}
       />
       <Virtuoso
         ref={listHandle}
         style={{ height: "100%" }}
         data={items}
         defaultItemHeight={50}
-        itemContent={(_index: number, data: ListItem) => {
-          return renderRow(data);
+        itemContent={renderRow}
+        followOutput={(_atBottom: boolean) => {
+          return followOutput;
         }}
-        followOutput={followOutput}
-        atBottomStateChange={(atBottom: boolean) => {
-          setFollowOutput(atBottom);
+        atBottomStateChange={handleAtBottomStateChange}
+        atBottomThreshold={30}
+        increaseViewportBy={{ top: 300, bottom: 300 }}
+        overscan={{
+          main: 10,
+          reverse: 10,
         }}
         className={clsx(className)}
         onKeyDown={onkeydown}
         skipAnimationFrameInResizeObserver={true}
+        isScrolling={isScrolling}
+        restoreStateFrom={getRestoreState()}
+      />
+      <SampleFooter
+        sampleCount={sampleCount}
+        totalSampleCount={totalItemCount}
+        running={running}
       />
-      <SampleFooter sampleCount={sampleCount} />
     </div>
   );
-};
+});
 
-const gridColumnsValue = (sampleDescriptor: SamplesDescriptor) => {
+const gridColumnsValue = (sampleDescriptor?: SamplesDescriptor) => {
   const { input, target, answer, limit, id, score } =
     gridColumns(sampleDescriptor);
   return `${id} ${input} ${target} ${answer} ${limit} ${score}`;
 };
 
-const gridColumns = (sampleDescriptor: SamplesDescriptor) => {
+const gridColumns = (sampleDescriptor?: SamplesDescriptor) => {
   const input =
-    sampleDescriptor?.messageShape.normalized.input > 0
+    sampleDescriptor && sampleDescriptor.messageShape.normalized.input > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.input)
       : 0;
   const target =
-    sampleDescriptor?.messageShape.normalized.target > 0
+    sampleDescriptor && sampleDescriptor.messageShape.normalized.target > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.target)
       : 0;
   const answer =
-    sampleDescriptor?.messageShape.normalized.answer > 0
+    sampleDescriptor && sampleDescriptor.messageShape.normalized.answer > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.answer)
       : 0;
   const limit =
-    sampleDescriptor?.messageShape.normalized.limit > 0
+    sampleDescriptor && sampleDescriptor.messageShape.normalized.limit > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.limit)
       : 0;
-  const id = Math.max(2, Math.min(10, sampleDescriptor?.messageShape.raw.id));
+  const id = Math.max(
+    2,
+    Math.min(10, sampleDescriptor?.messageShape.raw.id || 0),
+  );
   const score = Math.max(
     3,
-    Math.min(10, sampleDescriptor?.messageShape.raw.score),
+    Math.min(10, sampleDescriptor?.messageShape.raw.score || 0),
   );
 
   const frSize = (val: number) => {

inspect_ai/_view/www/src/samples/list/SampleRow.module.css

@@ -31,3 +31,9 @@
   display: flex;
   justify-self: center;
 }
+
+.spinner {
+  height: 1.4em;
+  width: 1.4em;
+  color: var(--bs-focus-ring-color);
+}

inspect_ai/_view/www/src/samples/list/SampleRow.tsx

@@ -1,9 +1,10 @@
 import clsx from "clsx";
-import { FC } from "react";
+import { FC, ReactNode, useCallback } from "react";
 import { SampleSummary } from "../../api/types";
 import { MarkdownDiv } from "../../components/MarkdownDiv";
+import { PulsingDots } from "../../components/PulsingDots";
+import { useStore } from "../../state/store";
 import { arrayToString, inputString } from "../../utils/format";
-import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
 import { SampleErrorView } from "../error/SampleErrorView";
 import styles from "./SampleRow.module.css";
 
@@ -11,10 +12,11 @@ interface SampleRowProps {
   id: string;
   index: number;
   sample: SampleSummary;
-  sampleDescriptor: SamplesDescriptor;
+  answer: string;
+  completed: boolean;
+  scoreRendered: ReactNode;
   gridColumnsTemplate: string;
   height: number;
-  selected: boolean;
   showSample: (index: number) => void;
 }
 
@@ -22,22 +24,33 @@ export const SampleRow: FC<SampleRowProps> = ({
   id,
   index,
   sample,
-  sampleDescriptor,
+  answer,
+  completed,
+  scoreRendered,
   gridColumnsTemplate,
   height,
-  selected,
   showSample,
 }) => {
+  const streamSampleData = useStore(
+    (state) => state.capabilities.streamSampleData,
+  );
+  const selectedSampleIndex = useStore(
+    (state) => state.log.selectedSampleIndex,
+  );
+  const handleClick = useCallback(() => {
+    if (completed || streamSampleData) {
+      showSample(index);
+    }
+  }, [index, showSample, completed]);
+
   return (
     <div
       id={`sample-${id}`}
-      onClick={() => {
-        showSample(index);
-      }}
+      onClick={handleClick}
       className={clsx(
         styles.grid,
         "text-size-base",
-        selected ? styles.selected : undefined,
+        selectedSampleIndex === index ? styles.selected : undefined,
       )}
       style={{
         height: `${height}px`,
@@ -56,7 +69,7 @@ export const SampleRow: FC<SampleRowProps> = ({
           styles.wrapAnywhere,
         )}
       >
-        {inputString(sample.input).join(" ")}
+        <MarkdownDiv markdown={inputString(sample.input).join(" ")} />
       </div>
       <div className={clsx("sample-target", "three-line-clamp", styles.cell)}>
         <MarkdownDiv
@@ -67,9 +80,7 @@ export const SampleRow: FC<SampleRowProps> = ({
       <div className={clsx("sample-answer", "three-line-clamp", styles.cell)}>
         {sample ? (
           <MarkdownDiv
-            markdown={sampleDescriptor
-              ?.selectedScorerDescriptor(sample)
-              .answer()}
+            markdown={answer || ""}
             className={clsx("no-last-para-padding", styles.noLeft)}
           />
         ) : (
@@ -90,8 +101,10 @@ export const SampleRow: FC<SampleRowProps> = ({
       <div className={clsx("text-size-small", styles.cell, styles.score)}>
         {sample.error ? (
           <SampleErrorView message={sample.error} />
+        ) : completed ? (
+          scoreRendered
         ) : (
-          sampleDescriptor?.selectedScore(sample)?.render()
+          <PulsingDots />
         )}
       </div>
     </div>

inspect_ai/_view/www/src/samples/sample-tools/SelectScorer.tsx

@@ -1,7 +1,7 @@
 import clsx from "clsx";
 import { ScoreLabel } from "../../types";
 
-import { FC } from "react";
+import { ChangeEvent, FC, useCallback } from "react";
 import styles from "./SelectScorer.module.css";
 
 interface SelectScorerProps {
@@ -26,6 +26,13 @@ export const SelectScorer: FC<SelectScorerProps> = ({
     return accum;
   }, [] as ScoreLabel[]);
 
+  const handleSelectScore = useCallback(
+    (index: number) => {
+      setScore(scores[index]);
+    },
+    [setScore, scores],
+  );
+
   if (scorers.length === 1) {
     // There is only a single scorer in play, just show the list of available scores
     return (
@@ -44,9 +51,7 @@ export const SelectScorer: FC<SelectScorerProps> = ({
         <ScoreSelector
           scores={scores}
           selectedIndex={scoreIndex(scores, score)}
-          setSelectedIndex={(index: number) => {
-            setScore(scores[index]);
-          }}
+          setSelectedIndex={handleSelectScore}
         />
       </div>
     );
@@ -75,18 +80,14 @@ export const SelectScorer: FC<SelectScorerProps> = ({
         <ScorerSelector
           scorers={scorers}
           selectedIndex={scorerIndex(scorers, score)}
-          setSelectedIndex={(index: number) => {
-            setScore(scorers[index]);
-          }}
+          setSelectedIndex={handleSelectScore}
         />
         {scorerScores.length > 1 ? (
           <ScoreSelector
             className={clsx(styles.secondSel)}
             scores={scorerScores}
             selectedIndex={scoreIndex(scorerScores, score)}
-            setSelectedIndex={(index: number) => {
-              setScore(scorerScores[index]);
-            }}
+            setSelectedIndex={handleSelectScore}
           />
         ) : undefined}
       </div>
@@ -107,6 +108,14 @@ const ScoreSelector: FC<ScoreSelectorProps> = ({
   setSelectedIndex,
   className,
 }) => {
+  const handleChange = useCallback(
+    (e: ChangeEvent<HTMLSelectElement>) => {
+      const sel = e.target as HTMLSelectElement;
+      setSelectedIndex(sel.selectedIndex);
+    },
+    [setSelectedIndex],
+  );
+
   return (
     <select
       className={clsx(
@@ -117,10 +126,7 @@ const ScoreSelector: FC<ScoreSelectorProps> = ({
       )}
       aria-label=".select-scorer-label"
       value={scores[selectedIndex].name}
-      onChange={(e) => {
-        const sel = e.target as HTMLSelectElement;
-        setSelectedIndex(sel.selectedIndex);
-      }}
+      onChange={handleChange}
     >
       {scores.map((score) => {
         return (
@@ -144,15 +150,20 @@ const ScorerSelector: FC<ScorerSelectorProps> = ({
   selectedIndex,
   setSelectedIndex,
 }) => {
+  const handleChange = useCallback(
+    (e: ChangeEvent<HTMLSelectElement>) => {
+      const sel = e.target as HTMLSelectElement;
+      setSelectedIndex(sel.selectedIndex);
+    },
+    [setSelectedIndex],
+  );
+
   return (
     <select
       className={clsx("form-select", "form-select-sm", "text-size-smaller")}
       aria-label=".epoch-filter-label"
       value={scorers[selectedIndex].scorer}
-      onChange={(e) => {
-        const sel = e.target as HTMLSelectElement;
-        setSelectedIndex(sel.selectedIndex);
-      }}
+      onChange={handleChange}
     >
       {scorers.map((scorer) => {
         return (

inspect_ai/_view/www/src/samples/sample-tools/SortFilter.tsx

@@ -1,5 +1,5 @@
 import clsx from "clsx";
-import { FC } from "react";
+import { ChangeEvent, FC, useCallback } from "react";
 import { SampleSummary } from "../../api/types";
 import {
   kEpochAscVal,
@@ -9,6 +9,7 @@ import {
   kScoreAscVal,
   kScoreDescVal,
 } from "../../constants";
+import { ScoreLabel } from "../../types";
 import { isNumeric } from "../../utils/type";
 import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
 import styles from "./SortFilter.module.css";
@@ -42,6 +43,15 @@ export const SortFilter: FC<SortFilterProps> = ({ sort, setSort, epochs }) => {
     label: "score desc",
     val: kScoreDescVal,
   });
+
+  const handleChange = useCallback(
+    (e: ChangeEvent<HTMLSelectElement>) => {
+      const sel = e.target as HTMLSelectElement;
+      setSort(sel.value);
+    },
+    [setSort],
+  );
+
   return (
     <div className={styles.flex}>
       <span
@@ -59,10 +69,7 @@ export const SortFilter: FC<SortFilterProps> = ({ sort, setSort, epochs }) => {
         className={clsx("form-select", "form-select-sm", "text-size-smaller")}
         aria-label=".sort-filter-label"
         value={sort}
-        onChange={(e) => {
-          const sel = e.target as HTMLSelectElement;
-          setSort(sel.value);
-        }}
+        onChange={handleChange}
       >
         {options.map((option) => {
           return (
@@ -102,8 +109,13 @@ export const sortSamples = (
   sort: string,
   samples: SampleSummary[],
   samplesDescriptor: SamplesDescriptor,
-): { sorted: SampleSummary[]; order: "asc" | "desc" } => {
+  score?: ScoreLabel,
+): SampleSummary[] => {
   const sortedSamples = samples.sort((a: SampleSummary, b: SampleSummary) => {
+    const scoreDescriptor = score
+      ? samplesDescriptor.evalDescriptor.scoreDescriptor(score)
+      : undefined;
+
     switch (sort) {
       case kSampleAscVal: {
         const result = sortId(a, b);
@@ -134,50 +146,38 @@ export const sortSamples = (
         if (result !== 0) {
           return result;
         } else {
-          return sortId(a, b);
+          return sortId(b, a);
         }
       }
 
       case kScoreAscVal: {
-        const aScore = samplesDescriptor.selectedScore(a);
-        const bScore = samplesDescriptor.selectedScore(b);
+        const aScore = samplesDescriptor.evalDescriptor.score(a, score);
+        const bScore = samplesDescriptor.evalDescriptor.score(b, score);
         if (
           aScore === undefined ||
           bScore === undefined ||
-          samplesDescriptor.selectedScoreDescriptor == undefined
+          scoreDescriptor === undefined
         ) {
           return 0;
         }
-        return samplesDescriptor.selectedScoreDescriptor.compare(
-          aScore,
-          bScore,
-        );
+        return scoreDescriptor?.compare(aScore, bScore);
       }
       case kScoreDescVal: {
-        const aScore = samplesDescriptor.selectedScore(a);
-        const bScore = samplesDescriptor.selectedScore(b);
+        const aScore = samplesDescriptor.evalDescriptor.score(a, score);
+        const bScore = samplesDescriptor.evalDescriptor.score(b, score);
         if (
           aScore === undefined ||
           bScore === undefined ||
-          samplesDescriptor.selectedScoreDescriptor == undefined
+          scoreDescriptor == undefined
         ) {
           return 0;
         }
 
-        return samplesDescriptor.selectedScoreDescriptor.compare(
-          bScore,
-          aScore,
-        );
+        return scoreDescriptor?.compare(bScore, aScore);
       }
       default:
         return 0;
     }
   });
-  return {
-    sorted: sortedSamples,
-    order:
-      sort === kSampleAscVal || sort === kEpochAscVal || sort === kScoreAscVal
-        ? "asc"
-        : "desc",
-  };
+  return sortedSamples;
 };