inspect-ai 0.3.80__py3-none-any.whl → 0.3.82__py3-none-any.whl

inspect_ai/_view/www/src/samples/descriptor/samplesDescriptor.tsx

@@ -15,14 +15,14 @@ import {
 export interface SamplesDescriptor {
   evalDescriptor: EvalDescriptor;
   messageShape: MessageShape;
-  selectedScoreDescriptor?: ScoreDescriptor;
   selectedScore: (sample: BasicSampleData) => SelectedScore | undefined;
-  selectedScorerDescriptor: (sample: BasicSampleData) => ScorerDescriptor;
+  selectedScorerDescriptor: (
+    sample: BasicSampleData,
+  ) => ScorerDescriptor | undefined;
 }
 
 export const createEvalDescriptor = (
   scores: ScoreLabel[],
-  epochs: number,
   samples?: SampleSummary[],
 ): EvalDescriptor | undefined => {
   if (!samples) {
@@ -57,10 +57,10 @@ export const createEvalDescriptor = (
 
   const scoreAnswer = (
     sample: BasicSampleData,
-    scorer: string,
+    scorer: ScoreLabel,
   ): string | undefined => {
     if (sample && sample.scores) {
-      const sampleScore = sample.scores[scorer];
+      const sampleScore = sample.scores[scorer.name];
       if (sampleScore && sampleScore.answer) {
         return sampleScore.answer;
       }
@@ -181,7 +181,7 @@ export const createEvalDescriptor = (
         return scoreExplanation(sample, scoreLabel.scorer) || "";
       },
       answer: () => {
-        return scoreAnswer(sample, scoreLabel.scorer) || "";
+        return scoreAnswer(sample, scoreLabel) || "";
       },
       scores: () => {
         if (!sample || !sample.scores) {
@@ -252,8 +252,11 @@ export const createEvalDescriptor = (
 
   const score = (
     sample: BasicSampleData,
-    scoreLabel: ScoreLabel,
-  ): SelectedScore => {
+    scoreLabel?: ScoreLabel,
+  ): SelectedScore | undefined => {
+    if (!scoreLabel) {
+      return undefined;
+    }
     return {
       value: scoreValue(sample, scoreLabel),
       render: () => {
@@ -263,8 +266,6 @@ export const createEvalDescriptor = (
   };
 
   return {
-    epochs,
-    samples,
     scores,
     scorerDescriptor,
     scoreDescriptor,
@@ -274,14 +275,17 @@ export const createEvalDescriptor = (
 };
 
 export const createSamplesDescriptor = (
+  samples: SampleSummary[],
   evalDescriptor: EvalDescriptor,
-  selectedScore: ScoreLabel,
+  selectedScore?: ScoreLabel,
 ): SamplesDescriptor | undefined => {
   // Find the total length of the value so we can compute an average
-  const sizes = evalDescriptor.samples.reduce(
+  const sizes = samples.reduce(
     (previous, current) => {
       const text = inputString(current.input).join(" ");
-      const score = evalDescriptor.score(current, selectedScore);
+      const score = selectedScore
+        ? evalDescriptor.score(current, selectedScore)
+        : undefined;
       const scoreValue = score?.value;
       const scoreText = scoreValue
         ? String(scoreValue)
@@ -296,7 +300,9 @@ export const createSamplesDescriptor = (
       previous[2] = Math.min(
         Math.max(
           previous[2],
-          evalDescriptor.scoreAnswer(current, selectedScore?.name)?.length || 0,
+          selectedScore
+            ? evalDescriptor.scoreAnswer(current, selectedScore)?.length || 0
+            : 0,
         ),
         300,
       );
@@ -353,10 +359,12 @@ export const createSamplesDescriptor = (
   return {
     evalDescriptor,
     messageShape,
-    selectedScoreDescriptor: evalDescriptor.scoreDescriptor(selectedScore),
-    selectedScore: (sample) => evalDescriptor.score(sample, selectedScore),
+    selectedScore: (sample) =>
+      selectedScore ? evalDescriptor.score(sample, selectedScore) : undefined,
     selectedScorerDescriptor: (sample) =>
-      evalDescriptor.scorerDescriptor(sample, selectedScore),
+      selectedScore
+        ? evalDescriptor.scorerDescriptor(sample, selectedScore)
+        : undefined,
   };
 };
 

inspect_ai/_view/www/src/samples/descriptor/score/ObjectScoreDescriptor.tsx

@@ -52,13 +52,14 @@ export const objectScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
             : String(value);
         scores.push(
           <div
+            key={`score-value-${index}`}
             className={clsx(
               styles.container,
               index + 1 < keys.length ? styles.padded : undefined,
             )}
           >
             <div className={clsx(styles.key, "text-size-smaller")}>{key}</div>
-            <div className={clsx(styles.value, "text-size-title")}>
+            <div className={clsx(styles.value, "text-size-large")}>
               {formattedValue}
             </div>
           </div>,

inspect_ai/_view/www/src/samples/descriptor/types.ts

@@ -1,11 +1,9 @@
 import { ReactNode } from "react";
-import { BasicSampleData, SampleSummary } from "../../api/types";
+import { BasicSampleData } from "../../api/types";
 import { ScoreLabel } from "../../types";
 import { Value2 } from "../../types/log";
 
 export interface EvalDescriptor {
-  epochs: number;
-  samples: SampleSummary[];
   scores: ScoreLabel[];
   scoreDescriptor: (scoreLabel: ScoreLabel) => ScoreDescriptor;
   scorerDescriptor: (
@@ -14,9 +12,12 @@ export interface EvalDescriptor {
   ) => ScorerDescriptor;
   score: (
     sample: BasicSampleData,
-    scoreLabel: ScoreLabel,
+    scoreLabel?: ScoreLabel,
   ) => SelectedScore | undefined;
-  scoreAnswer: (sample: BasicSampleData, scorer: string) => string | undefined;
+  scoreAnswer: (
+    sample: BasicSampleData,
+    scorer: ScoreLabel,
+  ) => string | undefined;
 }
 
 export interface ScorerDescriptor {

inspect_ai/_view/www/src/samples/list/SampleFooter.module.css

@@ -2,8 +2,26 @@
   border-top: solid var(--bs-light-border-subtle) 1px;
   background: var(--bs-light-bg-subtle);
   display: grid;
-  grid-template-columns: max-content;
-  justify-content: end;
-  align-content: end;
+  grid-template-columns: max-content max-content;
+  justify-content: space-between;
+
   padding: 0.2em 1em;
 }
+
+.spinnerContainer {
+  display: grid;
+  grid-template-columns: max-content max-content;
+  column-gap: 0.3em;
+  padding-top: 0.2em;
+}
+
+.spinner {
+  height: 11px;
+  width: 11px;
+  color: var(--bs-secondary);
+  border-width: 1px;
+}
+
+.label {
+  margin-top: -4px;
+}

inspect_ai/_view/www/src/samples/list/SampleFooter.tsx

@@ -1,14 +1,33 @@
 interface SampleFooterProps {
   sampleCount: number;
+  running: boolean;
 }
 
 import clsx from "clsx";
 import { FC } from "react";
 import styles from "./SampleFooter.module.css";
 
-export const SampleFooter: FC<SampleFooterProps> = ({ sampleCount }) => {
+export const SampleFooter: FC<SampleFooterProps> = ({
+  sampleCount,
+  running,
+}) => {
   return (
     <div className={clsx("text-size-smaller", styles.footer)}>
+      <div>
+        {running ? (
+          <div className={clsx(styles.spinnerContainer)}>
+            <div
+              className={clsx("spinner-border", styles.spinner)}
+              role="status"
+            >
+              <span className={clsx("visually-hidden")}>Running...</span>
+            </div>
+            <div className={clsx("text-style-secondary", styles.label)}>
+              running...
+            </div>
+          </div>
+        ) : undefined}
+      </div>
       <div>{sampleCount} Samples</div>
     </div>
   );

inspect_ai/_view/www/src/samples/list/SampleList.tsx

@@ -1,15 +1,14 @@
 import {
   FC,
   KeyboardEvent,
+  memo,
   RefObject,
   useCallback,
   useEffect,
   useMemo,
   useRef,
-  useState,
 } from "react";
 import { Virtuoso, VirtuosoHandle } from "react-virtuoso";
-import { EmptyPanel } from "../../components/EmptyPanel";
 import { MessageBand } from "../../components/MessageBand";
 import { formatNoDecimal } from "../../utils/format";
 import { ListItem } from "../../workspace/tabs/types";
@@ -18,6 +17,9 @@ import { SampleRow } from "./SampleRow";
 import { SampleSeparator } from "./SampleSeparator";
 
 import clsx from "clsx";
+import { useProperty, useSampleDescriptor } from "../../state/hooks";
+import { useVirtuosoState } from "../../state/scrolling";
+import { useStore } from "../../state/store";
 import { SampleFooter } from "./SampleFooter";
 import { SampleHeader } from "./SampleHeader";
 import styles from "./SampleList.module.css";
@@ -27,8 +29,7 @@ const kSeparatorHeight = 24;
 
 interface SampleListProps {
   items: ListItem[];
-  sampleDescriptor: SamplesDescriptor;
-  selectedIndex: number;
+  running: boolean;
   nextSample: () => void;
   prevSample: () => void;
   showSample: (index: number) => void;
@@ -36,11 +37,10 @@ interface SampleListProps {
   listHandle: RefObject<VirtuosoHandle | null>;
 }
 
-export const SampleList: FC<SampleListProps> = (props) => {
+export const SampleList: FC<SampleListProps> = memo((props) => {
   const {
     items,
-    sampleDescriptor,
-    selectedIndex,
+    running,
     nextSample,
     prevSample,
     showSample,
@@ -48,37 +48,50 @@ export const SampleList: FC<SampleListProps> = (props) => {
     listHandle,
   } = props;
 
-  const [followOutput, setFollowOutput] = useState(false);
+  const { getRestoreState, isScrolling } = useVirtuosoState(
+    listHandle,
+    "sample-list",
+  );
 
-  const [hidden, setHidden] = useState(false);
-  useEffect(() => {
-    setHidden(false);
-  }, [items]);
+  const selectedSampleIndex = useStore(
+    (state) => state.log.selectedSampleIndex,
+  );
+  const samplesDescriptor = useSampleDescriptor();
+  const [followOutput, setFollowOutput] = useProperty("sample-list", "follow", {
+    defaultValue: false,
+  });
 
-  // Keep a mapping of the indexes to items (skipping separators)
-  const itemRowMapping = useMemo(() => {
-    const rowIndexes: number[] = [];
-    items.forEach((item, index) => {
-      if (item.type === "sample") {
-        rowIndexes.push(index);
-      }
-    });
-    return rowIndexes;
-  }, [items]);
+  // Track whether we were previously running so we can
+  // decide whether to pop up to the top
+  const prevRunningRef = useRef(running);
 
-  const prevSelectedIndexRef = useRef<number>(null);
   useEffect(() => {
-    const listEl = listHandle.current;
-    if (listEl) {
-      requestAnimationFrame(() => {
-        setTimeout(() => {
-          const actualRowIndex = itemRowMapping[selectedIndex];
-          listEl.scrollToIndex(actualRowIndex);
-          prevSelectedIndexRef.current = actualRowIndex;
-        }, 10);
-      });
+    // When we finish running, if we are following output
+    // then scroll up to the top
+    if (
+      !running &&
+      prevRunningRef.current &&
+      followOutput &&
+      listHandle.current
+    ) {
+      setFollowOutput(false);
+      setTimeout(() => {
+        if (listHandle.current) {
+          listHandle.current.scrollTo({ top: 0, behavior: "instant" });
+        }
+      }, 100);
     }
-  }, [selectedIndex, listHandle, itemRowMapping]);
+    prevRunningRef.current = running;
+  }, [running, followOutput, listHandle]);
+
+  const handleAtBottomStateChange = useCallback(
+    (atBottom: boolean) => {
+      if (running) {
+        setFollowOutput(atBottom);
+      }
+    },
+    [running, setFollowOutput],
+  );
 
   const onkeydown = useCallback(
     (e: KeyboardEvent<HTMLDivElement>) => {
@@ -94,48 +107,51 @@ export const SampleList: FC<SampleListProps> = (props) => {
           e.stopPropagation();
           break;
         case "Enter":
-          showSample(selectedIndex);
+          showSample(selectedSampleIndex);
           e.preventDefault();
           e.stopPropagation();
           break;
       }
     },
-    [selectedIndex, nextSample, prevSample, showSample],
+    [selectedSampleIndex, nextSample, prevSample, showSample],
   );
 
-  // If there are no samples, just display an empty state
-  if (items.length === 0) {
-    return <EmptyPanel>No Samples</EmptyPanel>;
-  }
+  const gridColumnsTemplate = useMemo(() => {
+    return gridColumnsValue(samplesDescriptor);
+  }, [samplesDescriptor]);
 
-  const renderRow = (item: ListItem) => {
-    if (item.type === "sample") {
-      return (
-        <SampleRow
-          id={`${item.number}`}
-          index={item.index}
-          sample={item.data}
-          height={kSampleHeight}
-          sampleDescriptor={sampleDescriptor}
-          gridColumnsTemplate={gridColumnsValue(sampleDescriptor)}
-          selected={selectedIndex === item.index}
-          showSample={showSample}
-        />
-      );
-    } else if (item.type === "separator") {
-      return (
-        <SampleSeparator
-          id={`sample-group${item.number}`}
-          title={item.data}
-          height={kSeparatorHeight}
-        />
-      );
-    } else {
-      return null;
-    }
-  };
+  const renderRow = useCallback(
+    (_index: number, item: ListItem) => {
+      if (item.type === "sample") {
+        return (
+          <SampleRow
+            id={`${item.number}`}
+            index={item.index}
+            sample={item.data}
+            height={kSampleHeight}
+            answer={item.answer}
+            completed={item.completed}
+            scoreRendered={item.scoreRendered}
+            gridColumnsTemplate={gridColumnsTemplate}
+            showSample={showSample}
+          />
+        );
+      } else if (item.type === "separator") {
+        return (
+          <SampleSeparator
+            id={`sample-group${item.number}`}
+            title={item.data}
+            height={kSeparatorHeight}
+          />
+        );
+      } else {
+        return null;
+      }
+    },
+    [showSample],
+  );
 
-  const { input, limit, answer, target } = gridColumns(sampleDescriptor);
+  const { input, limit, answer, target } = gridColumns(samplesDescriptor);
 
   const sampleCount = items?.reduce((prev, current) => {
     if (current.type === "sample") {
@@ -176,68 +192,72 @@ export const SampleList: FC<SampleListProps> = (props) => {
     <div className={styles.mainLayout}>
       {warningMessage ? (
         <MessageBand
+          id={"sample-warning-message"}
           message={warningMessage}
-          hidden={hidden}
-          setHidden={setHidden}
           type="info"
         />
       ) : undefined}
-
       <SampleHeader
         input={input !== "0"}
         target={target !== "0"}
         answer={answer !== "0"}
         limit={limit !== "0"}
-        gridColumnsTemplate={gridColumnsValue(sampleDescriptor)}
+        gridColumnsTemplate={gridColumnsTemplate}
       />
       <Virtuoso
         ref={listHandle}
         style={{ height: "100%" }}
         data={items}
         defaultItemHeight={50}
-        itemContent={(_index: number, data: ListItem) => {
-          return renderRow(data);
-        }}
+        itemContent={renderRow}
         followOutput={followOutput}
-        atBottomStateChange={(atBottom: boolean) => {
-          setFollowOutput(atBottom);
+        atBottomStateChange={handleAtBottomStateChange}
+        increaseViewportBy={{ top: 300, bottom: 300 }}
+        overscan={{
+          main: 10,
+          reverse: 10,
         }}
         className={clsx(className)}
         onKeyDown={onkeydown}
         skipAnimationFrameInResizeObserver={true}
+        isScrolling={isScrolling}
+        restoreStateFrom={getRestoreState()}
       />
-      <SampleFooter sampleCount={sampleCount} />
+      <SampleFooter sampleCount={sampleCount} running={running} />
     </div>
   );
-};
+});
 
-const gridColumnsValue = (sampleDescriptor: SamplesDescriptor) => {
+const gridColumnsValue = (sampleDescriptor?: SamplesDescriptor) => {
   const { input, target, answer, limit, id, score } =
     gridColumns(sampleDescriptor);
   return `${id} ${input} ${target} ${answer} ${limit} ${score}`;
 };
 
-const gridColumns = (sampleDescriptor: SamplesDescriptor) => {
+const gridColumns = (sampleDescriptor?: SamplesDescriptor) => {
   const input =
-    sampleDescriptor?.messageShape.normalized.input > 0
+    sampleDescriptor && sampleDescriptor.messageShape.normalized.input > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.input)
       : 0;
   const target =
-    sampleDescriptor?.messageShape.normalized.target > 0
+    sampleDescriptor && sampleDescriptor.messageShape.normalized.target > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.target)
       : 0;
   const answer =
-    sampleDescriptor?.messageShape.normalized.answer > 0
+    sampleDescriptor && sampleDescriptor.messageShape.normalized.answer > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.answer)
       : 0;
   const limit =
-    sampleDescriptor?.messageShape.normalized.limit > 0
+    sampleDescriptor && sampleDescriptor.messageShape.normalized.limit > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.limit)
       : 0;
-  const id = Math.max(2, Math.min(10, sampleDescriptor?.messageShape.raw.id));
+  const id = Math.max(
+    2,
+    Math.min(10, sampleDescriptor?.messageShape.raw.id || 0),
+  );
   const score = Math.max(
     3,
-    Math.min(10, sampleDescriptor?.messageShape.raw.score),
+    Math.min(10, sampleDescriptor?.messageShape.raw.score || 0),
   );
 
   const frSize = (val: number) => {

inspect_ai/_view/www/src/samples/list/SampleRow.module.css

@@ -31,3 +31,9 @@
   display: flex;
   justify-self: center;
 }
+
+.spinner {
+  height: 1.4em;
+  width: 1.4em;
+  color: var(--bs-focus-ring-color);
+}

inspect_ai/_view/www/src/samples/list/SampleRow.tsx

@@ -1,9 +1,10 @@
 import clsx from "clsx";
-import { FC } from "react";
+import { FC, ReactNode, useCallback } from "react";
 import { SampleSummary } from "../../api/types";
 import { MarkdownDiv } from "../../components/MarkdownDiv";
+import { PulsingDots } from "../../components/PulsingDots";
+import { useStore } from "../../state/store";
 import { arrayToString, inputString } from "../../utils/format";
-import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
 import { SampleErrorView } from "../error/SampleErrorView";
 import styles from "./SampleRow.module.css";
 
@@ -11,10 +12,11 @@ interface SampleRowProps {
   id: string;
   index: number;
   sample: SampleSummary;
-  sampleDescriptor: SamplesDescriptor;
+  answer: string;
+  completed: boolean;
+  scoreRendered: ReactNode;
   gridColumnsTemplate: string;
   height: number;
-  selected: boolean;
   showSample: (index: number) => void;
 }
 
@@ -22,22 +24,33 @@ export const SampleRow: FC<SampleRowProps> = ({
   id,
   index,
   sample,
-  sampleDescriptor,
+  answer,
+  completed,
+  scoreRendered,
   gridColumnsTemplate,
   height,
-  selected,
   showSample,
 }) => {
+  const streamSampleData = useStore(
+    (state) => state.capabilities.streamSampleData,
+  );
+  const selectedSampleIndex = useStore(
+    (state) => state.log.selectedSampleIndex,
+  );
+  const handleClick = useCallback(() => {
+    if (completed || streamSampleData) {
+      showSample(index);
+    }
+  }, [index, showSample, completed]);
+
   return (
     <div
       id={`sample-${id}`}
-      onClick={() => {
-        showSample(index);
-      }}
+      onClick={handleClick}
       className={clsx(
         styles.grid,
         "text-size-base",
-        selected ? styles.selected : undefined,
+        selectedSampleIndex === index ? styles.selected : undefined,
       )}
       style={{
         height: `${height}px`,
@@ -67,9 +80,7 @@ export const SampleRow: FC<SampleRowProps> = ({
       <div className={clsx("sample-answer", "three-line-clamp", styles.cell)}>
         {sample ? (
           <MarkdownDiv
-            markdown={sampleDescriptor
-              ?.selectedScorerDescriptor(sample)
-              .answer()}
+            markdown={answer || ""}
             className={clsx("no-last-para-padding", styles.noLeft)}
           />
         ) : (
@@ -90,8 +101,10 @@ export const SampleRow: FC<SampleRowProps> = ({
       <div className={clsx("text-size-small", styles.cell, styles.score)}>
         {sample.error ? (
           <SampleErrorView message={sample.error} />
+        ) : completed ? (
+          scoreRendered
         ) : (
-          sampleDescriptor?.selectedScore(sample)?.render()
+          <PulsingDots />
         )}
       </div>
     </div>