inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/workspace/tabs/SamplesTab.tsx

@@ -1,135 +1,153 @@
 import {
   FC,
   Fragment,
-  RefObject,
   useCallback,
   useEffect,
+  useMemo,
   useRef,
   useState,
 } from "react";
 import { VirtuosoHandle } from "react-virtuoso";
-import { SampleSummary } from "../../api/types.ts";
-import { EmptyPanel } from "../../components/EmptyPanel.tsx";
-import { InlineSampleDisplay } from "../../samples/InlineSampleDisplay";
-import { SampleDialog } from "../../samples/SampleDialog";
-import { SamplesDescriptor } from "../../samples/descriptor/samplesDescriptor.tsx";
-import { SampleList } from "../../samples/list/SampleList";
-import { SampleMode, ScoreFilter } from "../../types.ts";
-import { EvalSample } from "../../types/log";
+import { NoContentsPanel } from "../../components/NoContentsPanel.tsx";
+import { InlineSampleDisplay } from "../../samples/InlineSampleDisplay.tsx";
+import { SampleDialog } from "../../samples/SampleDialog.tsx";
+import { SampleList } from "../../samples/list/SampleList.tsx";
+import {
+  useFilteredSamples,
+  useGroupBy,
+  useGroupByOrder,
+  useSampleDescriptor,
+  useScore,
+  useTotalSampleCount,
+} from "../../state/hooks.ts";
+import { useStore } from "../../state/store.ts";
+import { RunningNoSamples } from "./RunningNoSamples.tsx";
 import { getSampleProcessor } from "./grouping.ts";
 import { ListItem } from "./types.ts";
 
 interface SamplesTabProps {
-  // Optional props
-  sample?: EvalSample;
-  samples?: SampleSummary[];
-  sampleDescriptor?: SamplesDescriptor;
-  sampleError?: Error;
-
   // Required props
-  sampleMode: SampleMode;
-  groupBy: "epoch" | "sample" | "none";
-  groupByOrder: "asc" | "desc";
-  sampleStatus: string;
-  selectedSampleIndex: number;
-  setSelectedSampleIndex: (index: number) => void;
-  showingSampleDialog: boolean;
-  setShowingSampleDialog: (showing: boolean) => void;
-  selectedSampleTab?: string;
-  setSelectedSampleTab: (tab: string) => void;
-  epoch: string;
-  filter: ScoreFilter;
-  sampleScrollPositionRef: RefObject<number>;
-  setSampleScrollPosition: (position: number) => void;
-  sampleTabScrollRef: RefObject<HTMLDivElement | null>;
+  running: boolean;
 }
 
-export const SamplesTab: FC<SamplesTabProps> = ({
-  sample,
-  samples,
-  sampleMode,
-  groupBy,
-  groupByOrder,
-  sampleDescriptor,
-  sampleStatus,
-  sampleError,
-  selectedSampleIndex,
-  setSelectedSampleIndex,
-  showingSampleDialog,
-  setShowingSampleDialog,
-  selectedSampleTab,
-  setSelectedSampleTab,
-  sampleScrollPositionRef,
-  setSampleScrollPosition,
-  sampleTabScrollRef,
-}) => {
+export const SamplesTab: FC<SamplesTabProps> = ({ running }) => {
+  const selectSample = useStore((state) => state.logActions.selectSample);
+  const selectedSampleIndex = useStore(
+    (state) => state.log.selectedSampleIndex,
+  );
+
+  const sampleSummaries = useFilteredSamples();
+  const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
+
+  // Compute the limit to apply to the sample count (this is so)
+  // we can provide a total expected sample count for this evaluation
+  const evalSampleCount = useMemo(() => {
+    const limit = selectedLogSummary?.eval.config.limit;
+    const limitCount =
+      limit === null || limit === undefined
+        ? undefined
+        : typeof limit === "number"
+          ? limit
+          : (limit[1] as number) - (limit[0] as number);
+    return (
+      (limitCount || selectedLogSummary?.eval.dataset.samples || 0) *
+      (selectedLogSummary?.eval.config.epochs || 0)
+    );
+  }, [selectedLogSummary?.eval.config.limit]);
+
+  const totalSampleCount = useTotalSampleCount();
+
+  const samplesDescriptor = useSampleDescriptor();
+  const groupBy = useGroupBy();
+  const groupByOrder = useGroupByOrder();
+  const currentScore = useScore();
+
+  const selectedSample = useStore((state) => state.sample.selectedSample);
+
   const [items, setItems] = useState<ListItem[]>([]);
   const [sampleItems, setSampleItems] = useState<ListItem[]>([]);
 
   const sampleListHandle = useRef<VirtuosoHandle | null>(null);
   const sampleDialogRef = useRef<HTMLDivElement>(null);
 
+  const selectedSampleTab = useStore((state) => state.app.tabs.sample);
+  const setSelectedSampleTab = useStore(
+    (state) => state.appActions.setSampleTab,
+  );
+  const showingSampleDialog = useStore((state) => state.app.dialogs.sample);
+  const setShowingSampleDialog = useStore(
+    (state) => state.appActions.setShowingSampleDialog,
+  );
+
   // Shows the sample dialog
   const showSample = useCallback(
     (index: number) => {
-      setSelectedSampleIndex(index);
+      selectSample(index);
       setShowingSampleDialog(true);
     },
-    [setSelectedSampleIndex, setShowingSampleDialog],
+    [selectSample, setShowingSampleDialog],
   );
 
+  // Keep the selected item scrolled into view
+  useEffect(() => {
+    setTimeout(() => {
+      if (sampleListHandle.current) {
+        sampleListHandle.current.scrollIntoView({ index: selectedSampleIndex });
+      }
+    }, 0);
+  }, [selectedSampleIndex]);
+
+  // Focus the dialog when it is shown
   useEffect(() => {
     if (showingSampleDialog) {
       setTimeout(() => {
         sampleDialogRef.current?.focus();
       }, 0);
-    } else {
-      setTimeout(() => {
-        if (sampleListHandle.current) {
-          sampleListHandle.current.scrollToIndex(0);
-        }
-      }, 0);
     }
   }, [showingSampleDialog]);
 
+  const sampleProcessor = useMemo(() => {
+    if (!samplesDescriptor) return undefined;
+
+    return getSampleProcessor(
+      sampleSummaries || [],
+      selectedLogSummary?.eval?.config?.epochs || 1,
+      groupBy,
+      groupByOrder,
+      samplesDescriptor,
+      currentScore,
+    );
+  }, [
+    samplesDescriptor,
+    sampleSummaries,
+    selectedLogSummary?.eval?.config?.epochs,
+    groupBy,
+    groupByOrder,
+    currentScore,
+  ]);
+
   useEffect(() => {
-    const sampleProcessor = sampleDescriptor
-      ? getSampleProcessor(
-          samples || [],
-          groupBy,
-          groupByOrder,
-          sampleDescriptor,
-        )
-      : undefined;
-
-    // Process the samples into the proper data structure
-    const items = samples?.flatMap((sample, index) => {
+    const resolvedSamples = sampleSummaries?.flatMap((sample, index) => {
       const results: ListItem[] = [];
-      const previousSample = index !== 0 ? samples[index - 1] : undefined;
+      const previousSample =
+        index !== 0 ? sampleSummaries[index - 1] : undefined;
       const items = sampleProcessor
         ? sampleProcessor(sample, index, previousSample)
         : [];
+
       results.push(...items);
       return results;
     });
 
-    setItems(items || []);
+    setItems(resolvedSamples || []);
     setSampleItems(
-      items
-        ? items.filter((item) => {
+      resolvedSamples
+        ? resolvedSamples.filter((item) => {
             return item.type === "sample";
           })
         : [],
     );
-  }, [samples, groupBy, groupByOrder, sampleDescriptor]);
-
-  const nextSampleIndex = useCallback(() => {
-    if (selectedSampleIndex < sampleItems.length - 1) {
-      return selectedSampleIndex + 1;
-    } else {
-      return -1;
-    }
-  }, [selectedSampleIndex, sampleItems.length]);
+  }, [sampleSummaries, sampleProcessor]);
 
   const previousSampleIndex = useCallback(() => {
     return selectedSampleIndex > 0 ? selectedSampleIndex - 1 : -1;
@@ -137,68 +155,63 @@ export const SamplesTab: FC<SamplesTabProps> = ({
 
   // Manage the next / previous state the selected sample
   const nextSample = useCallback(() => {
-    const next = nextSampleIndex();
-    if (sampleStatus !== "loading" && next > -1) {
-      setSelectedSampleIndex(next);
+    const next = Math.min(selectedSampleIndex + 1, sampleItems.length - 1);
+    if (next > -1) {
+      selectSample(next);
     }
-  }, [nextSampleIndex, sampleStatus, setSelectedSampleIndex]);
+  }, [selectedSampleIndex, sampleItems, selectSample]);
 
   const previousSample = useCallback(() => {
     const prev = previousSampleIndex();
-    if (sampleStatus !== "loading" && prev > -1) {
-      setSelectedSampleIndex(prev);
+    if (prev > -1) {
+      selectSample(prev);
     }
-  }, [previousSampleIndex, sampleStatus, setSelectedSampleIndex]);
+  }, [previousSampleIndex, selectSample]);
 
   const title =
     selectedSampleIndex > -1 && sampleItems.length > selectedSampleIndex
       ? sampleItems[selectedSampleIndex].label
       : "";
 
-  if (!sampleDescriptor) {
-    return <EmptyPanel />;
+  if (totalSampleCount === 0) {
+    if (running) {
+      return <RunningNoSamples />;
+    } else {
+      return <NoContentsPanel text="No samples" />;
+    }
   } else {
     return (
       <Fragment>
-        {sampleDescriptor && sampleMode === "single" ? (
+        {samplesDescriptor && totalSampleCount === 1 ? (
           <InlineSampleDisplay
             id="sample-display"
-            sample={sample}
-            sampleStatus={sampleStatus}
-            sampleError={sampleError}
-            sampleDescriptor={sampleDescriptor}
             selectedTab={selectedSampleTab}
             setSelectedTab={setSelectedSampleTab}
-            scrollRef={sampleTabScrollRef}
           />
         ) : undefined}
-        {sampleDescriptor && sampleMode === "many" ? (
+        {samplesDescriptor && totalSampleCount > 1 ? (
           <SampleList
             listHandle={sampleListHandle}
             items={items}
-            sampleDescriptor={sampleDescriptor}
-            selectedIndex={selectedSampleIndex}
+            totalItemCount={evalSampleCount}
+            running={running}
             nextSample={nextSample}
             prevSample={previousSample}
             showSample={showSample}
           />
         ) : undefined}
-        <SampleDialog
-          id={String(sample?.id || "")}
-          title={title}
-          sample={sample}
-          sampleStatus={sampleStatus}
-          sampleError={sampleError}
-          sampleDescriptor={sampleDescriptor}
-          showingSampleDialog={showingSampleDialog}
-          setShowingSampleDialog={setShowingSampleDialog}
-          selectedTab={selectedSampleTab}
-          setSelectedTab={setSelectedSampleTab}
-          nextSample={nextSample}
-          prevSample={previousSample}
-          sampleScrollPositionRef={sampleScrollPositionRef}
-          setSampleScrollPosition={setSampleScrollPosition}
-        />
+        {showingSampleDialog ? (
+          <SampleDialog
+            id={String(selectedSample?.id || "")}
+            title={title}
+            showingSampleDialog={showingSampleDialog}
+            setShowingSampleDialog={setShowingSampleDialog}
+            selectedTab={selectedSampleTab}
+            setSelectedTab={setSelectedSampleTab}
+            nextSample={nextSample}
+            prevSample={previousSample}
+          />
+        ) : undefined}
       </Fragment>
     );
   }

inspect_ai/_view/www/src/workspace/tabs/grouping.ts

@@ -1,12 +1,16 @@
 import { SampleSummary } from "../../api/types";
 import { SamplesDescriptor } from "../../samples/descriptor/samplesDescriptor";
+import { ScoreLabel } from "../../types";
+import { Epochs } from "../../types/log";
 import { ListItem, SampleListItem, SeparatorListItem } from "./types";
 
 export const getSampleProcessor = (
   samples: SampleSummary[],
+  epochs: Epochs,
   groupBy: "sample" | "epoch" | "none",
   groupByOrder: "asc" | "desc",
   sampleDescriptor: SamplesDescriptor,
+  score?: ScoreLabel,
 ): ((
   sample: SampleSummary,
   index: number,
@@ -14,11 +18,17 @@ export const getSampleProcessor = (
 ) => ListItem[]) => {
   // Perform grouping if there are epochs
   if (groupBy == "epoch") {
-    return groupByEpoch(samples, sampleDescriptor, groupByOrder);
+    return groupByEpoch(samples, epochs, sampleDescriptor, groupByOrder, score);
   } else if (groupBy === "sample") {
-    return groupBySample(samples, sampleDescriptor, groupByOrder);
+    return groupBySample(
+      samples,
+      epochs,
+      sampleDescriptor,
+      groupByOrder,
+      score,
+    );
   } else {
-    return noGrouping(samples, groupByOrder);
+    return noGrouping(samples, groupByOrder, sampleDescriptor, score);
   }
 };
 
@@ -28,6 +38,8 @@ export const getSampleProcessor = (
 const noGrouping = (
   samples: SampleSummary[],
   order: "asc" | "desc",
+  sampleDescriptor: SamplesDescriptor,
+  score?: ScoreLabel,
 ): ((sample: SampleSummary, index: number) => ListItem[]) => {
   const counter = getCounter(samples.length, 1, order);
   return (sample: SampleSummary, index: number) => {
@@ -40,6 +52,12 @@ const noGrouping = (
         index: index,
         data: sample,
         type: "sample",
+        answer:
+          sampleDescriptor.selectedScorerDescriptor(sample)?.answer() || "",
+        scoreRendered: sampleDescriptor.evalDescriptor
+          .score(sample, score)
+          ?.render(),
+        completed: sample.completed !== undefined ? sample.completed : true,
       },
     ];
   };
@@ -50,8 +68,10 @@ const noGrouping = (
  */
 const groupBySample = (
   samples: SampleSummary[],
+  epochs: Epochs,
   sampleDescriptor: SamplesDescriptor,
   order: "asc" | "desc",
+  score?: ScoreLabel,
 ): ((
   sample: SampleSummary,
   index: number,
@@ -73,7 +93,7 @@ const groupBySample = (
       }
     }
   });
-  const groupCount = samples.length / sampleDescriptor.evalDescriptor.epochs;
+  const groupCount = samples.length / (epochs || 1);
   const itemCount = samples.length / groupCount;
   const counter = getCounter(itemCount, groupCount, order);
   return (
@@ -103,6 +123,11 @@ const groupBySample = (
       index: index,
       data: sample,
       type: "sample",
+      answer: sampleDescriptor.selectedScorerDescriptor(sample)?.answer() || "",
+      scoreRendered: sampleDescriptor.evalDescriptor
+        .score(sample, score)
+        ?.render(),
+      completed: sample.completed !== undefined ? sample.completed : true,
     } as SampleListItem);
 
     return results;
@@ -114,14 +139,16 @@ const groupBySample = (
  */
 const groupByEpoch = (
   samples: SampleSummary[],
+  epochs: Epochs,
   sampleDescriptor: SamplesDescriptor,
   order: "asc" | "desc",
+  score?: ScoreLabel,
 ): ((
   sample: SampleSummary,
   index: number,
   previousSample?: SampleSummary,
 ) => ListItem[]) => {
-  const groupCount = sampleDescriptor.evalDescriptor.epochs;
+  const groupCount = epochs || 1;
   const itemCount = samples.length / groupCount;
   const counter = getCounter(itemCount, groupCount, order);
 
@@ -153,6 +180,11 @@ const groupByEpoch = (
       index: index,
       data: sample,
       type: "sample",
+      answer: sampleDescriptor.selectedScorerDescriptor(sample)?.answer() || "",
+      scoreRendered: sampleDescriptor.evalDescriptor
+        .score(sample, score)
+        ?.render(),
+      completed: sample.completed !== undefined ? sample.completed : true,
     } as SampleListItem);
 
     return results;

inspect_ai/_view/www/src/workspace/tabs/types.ts

@@ -1,11 +1,15 @@
+import { ReactNode } from "react";
 import { SampleSummary } from "../../api/types";
 
 export interface SampleListItem {
   label: string;
   index: number;
   number: number;
+  answer: string;
+  scoreRendered: ReactNode;
   data: SampleSummary;
   type: "sample";
+  completed: boolean;
 }
 
 export interface SeparatorListItem {

inspect_ai/_view/www/src/workspace/types.ts

@@ -1,10 +1,11 @@
-import { ReactNode, RefObject } from "react";
+import { ComponentType, ReactNode, RefObject } from "react";
 
-export interface TabDescriptor {
+export interface TabDescriptor<P> {
   id: string;
   scrollable: boolean;
   scrollRef?: RefObject<HTMLDivElement | null>;
   label: string;
-  content: () => ReactNode;
+  component: ComponentType<P>;
+  componentProps: P;
   tools?: () => ReactNode[] | undefined;
 }

inspect_ai/_view/www/src/workspace/utils.ts

@@ -1,6 +1,6 @@
-import { EvalMetric } from "../types/log";
+import { ResultsMetric } from "./navbar/ResultsPanel";
 
-export const metricDisplayName = (metric: EvalMetric): string => {
+export const metricDisplayName = (metric: ResultsMetric): string => {
   let modifier = undefined;
   for (const metricModifier of metricModifiers) {
     modifier = metricModifier(metric);
@@ -13,10 +13,10 @@ export const metricDisplayName = (metric: EvalMetric): string => {
   return metricName;
 };
 
-type MetricModifier = (metric: EvalMetric) => string | undefined;
+type MetricModifier = (metric: ResultsMetric) => string | undefined;
 
 const clusterMetricModifier: MetricModifier = (
-  metric: EvalMetric,
+  metric: ResultsMetric,
 ): string | undefined => {
   if (metric.name !== "stderr") {
     return undefined;

inspect_ai/_view/www/vite.config.js

@@ -24,4 +24,10 @@ export default defineConfig({
   resolve: {
     dedupe: ["react", "react-dom"],
   },
+  define: {
+    __DEV_WATCH__: JSON.stringify(process.env.DEV_LOGGING === "true"),
+    __LOGGING_FILTER__: JSON.stringify(
+      process.env.DEV_LOGGING_NAMESPACES || "*",
+    ),
+  },
 });