inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/samples/SampleDialog.tsx

@@ -1,27 +1,22 @@
 import { ApplicationIcons } from "../appearance/icons";
 import { LargeModal, ModalTool, ModalTools } from "../components/LargeModal";
 
-import { FC, Ref, RefObject, useCallback, useMemo, useRef } from "react";
+import { FC, Ref, useCallback, useEffect, useMemo, useRef } from "react";
 import { ErrorPanel } from "../components/ErrorPanel";
-import { EvalSample } from "../types/log";
+import { useLogSelection, usePrevious, useSampleData } from "../state/hooks";
+import { useStatefulScrollPosition } from "../state/scrolling";
+import { useStore } from "../state/store";
 import { SampleDisplay } from "./SampleDisplay";
-import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
 
 interface SampleDialogProps {
   id: string;
   title: string;
-  sampleStatus: string;
-  sampleError?: Error;
-  sample?: EvalSample;
-  sampleDescriptor: SamplesDescriptor;
   selectedTab?: string;
   setSelectedTab: (tab: string) => void;
   showingSampleDialog: boolean;
   setShowingSampleDialog: (showing: boolean) => void;
   nextSample: () => void;
   prevSample: () => void;
-  sampleScrollPositionRef: RefObject<number>;
-  setSampleScrollPosition: (position: number) => void;
 }
 
 /**
@@ -30,21 +25,62 @@ interface SampleDialogProps {
 export const SampleDialog: FC<SampleDialogProps> = ({
   id,
   title,
-  sample,
-  sampleDescriptor,
   nextSample,
   prevSample,
-  sampleStatus,
-  sampleError,
   showingSampleDialog,
   setShowingSampleDialog,
   selectedTab,
   setSelectedTab,
-  sampleScrollPositionRef,
-  setSampleScrollPosition,
 }) => {
+  // Scroll referernce (attach stateful trackign)
   const scrollRef: Ref<HTMLDivElement> = useRef(null);
+  useStatefulScrollPosition(scrollRef, "sample-dialog");
 
+  // Sample hooks
+  const sampleData = useSampleData();
+  const loadSample = useStore((state) => state.sampleActions.loadSample);
+  const pollSample = useStore((state) => state.sampleActions.pollSample);
+  const logSelection = useLogSelection();
+
+  useEffect(() => {
+    if (sampleData.running && logSelection.logFile && logSelection.sample) {
+      pollSample(logSelection.logFile, logSelection.sample);
+    }
+  }, []);
+
+  // Load sample
+  const prevCompleted = usePrevious(
+    logSelection.sample?.completed !== undefined
+      ? logSelection.sample.completed
+      : true,
+  );
+  const prevLogFile = usePrevious<string | undefined>(logSelection.logFile);
+  useEffect(() => {
+    if (logSelection.logFile && logSelection.sample) {
+      const currentSampleCompleted =
+        logSelection.sample.completed !== undefined
+          ? logSelection.sample.completed
+          : true;
+
+      if (
+        prevLogFile !== logSelection.logFile ||
+        sampleData.sample?.id !== logSelection.sample.id ||
+        sampleData.sample?.epoch !== logSelection.sample.epoch ||
+        currentSampleCompleted !== prevCompleted
+      ) {
+        loadSample(logSelection.logFile, logSelection.sample);
+      }
+    }
+  }, [
+    logSelection.logFile,
+    logSelection.sample?.id,
+    logSelection.sample?.epoch,
+    logSelection.sample?.completed,
+    sampleData.sample?.id,
+    sampleData.sample?.epoch,
+  ]);
+
+  // Tools
   const tools = useMemo<ModalTools>(() => {
     const nextTool: ModalTool = {
       label: "Next Sample",
@@ -100,18 +136,18 @@ export const SampleDialog: FC<SampleDialogProps> = ({
       onkeyup={handleKeyUp}
       visible={showingSampleDialog}
       onHide={onHide}
-      showProgress={sampleStatus === "loading"}
-      initialScrollPositionRef={sampleScrollPositionRef}
-      setInitialScrollPosition={setSampleScrollPosition}
+      showProgress={
+        sampleData.status === "loading" || sampleData.status === "streaming"
+      }
       scrollRef={scrollRef}
     >
-      {sampleError ? (
-        <ErrorPanel title="Sample Error" error={sampleError} />
+      {sampleData.error ? (
+        <ErrorPanel title="Sample Error" error={sampleData.error} />
       ) : (
         <SampleDisplay
           id={id}
-          sample={sample}
-          sampleDescriptor={sampleDescriptor}
+          sample={sampleData.sample}
+          runningEvents={sampleData.running}
           selectedTab={selectedTab}
           setSelectedTab={setSelectedTab}
           scrollRef={scrollRef}

inspect_ai/_view/www/src/samples/SampleDisplay.module.css

@@ -35,3 +35,7 @@
   column-gap: 0.5em;
   min-width: 200px;
 }
+
+.chat {
+  padding-top: 1em;
+}

inspect_ai/_view/www/src/samples/SampleDisplay.tsx

@@ -7,13 +7,20 @@ import { isVscode } from "../utils/vscode";
 import { ApplicationIcons } from "../appearance/icons";
 import { ANSIDisplay } from "../components/AnsiDisplay";
 import { ToolButton } from "../components/ToolButton";
-import { SampleScoreView } from "./scores/SampleScoreView";
 
 import clsx from "clsx";
-import { FC, Fragment, MouseEvent, RefObject } from "react";
+import {
+  FC,
+  Fragment,
+  MouseEvent,
+  RefObject,
+  useCallback,
+  useMemo,
+} from "react";
+import { SampleSummary } from "../api/types";
 import { Card, CardBody, CardHeader } from "../components/Card";
-import { EmptyPanel } from "../components/EmptyPanel";
 import { JSONPanel } from "../components/JsonPanel";
+import { NoContentsPanel } from "../components/NoContentsPanel";
 import {
   kSampleErrorTabId,
   kSampleJsonTabId,
@@ -22,23 +29,26 @@ import {
   kSampleScoringTabId,
   kSampleTranscriptTabId,
 } from "../constants";
-import { EvalSample } from "../types/log";
+import { useSampleSummaries } from "../state/hooks";
+import { useStore } from "../state/store";
+import { EvalSample, Events } from "../types/log";
 import { ModelTokenTable } from "../usage/ModelTokenTable";
 import { formatTime } from "../utils/format";
 import { printHeadingHtml, printHtml } from "../utils/print";
 import { ChatViewVirtualList } from "./chat/ChatViewVirtualList";
-import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
+import { messagesFromEvents } from "./chat/messages";
 import styles from "./SampleDisplay.module.css";
 import { SampleSummaryView } from "./SampleSummaryView";
-import { SampleTranscript } from "./transcript/SampleTranscript";
+import { SampleScoresView } from "./scores/SampleScoresView";
+import { TranscriptVirtualList } from "./transcript/TranscriptView";
 
 interface SampleDisplayProps {
   id: string;
   sample?: EvalSample;
-  sampleDescriptor: SamplesDescriptor;
   selectedTab?: string;
   setSelectedTab: (tab: string) => void;
   scrollRef: RefObject<HTMLDivElement | null>;
+  runningEvents?: Events;
 }
 
 /**
@@ -47,18 +57,32 @@ interface SampleDisplayProps {
 export const SampleDisplay: FC<SampleDisplayProps> = ({
   id,
   sample,
-  sampleDescriptor,
   selectedTab,
   setSelectedTab,
   scrollRef,
+  runningEvents: runningSampleData,
 }) => {
   // Tab ids
   const baseId = `sample-dialog`;
+  const sampleSummaries = useSampleSummaries();
+  const selectedSampleIndex = useStore(
+    (state) => state.log.selectedSampleIndex,
+  );
 
-  if (!sample) {
-    // Placeholder
-    return <EmptyPanel />;
-  }
+  const sampleSummary = sampleSummaries[selectedSampleIndex];
+
+  // Consolidate the events and messages into the proper list
+  // whether running or not
+  const sampleEvents = sample?.events || runningSampleData;
+  const sampleMessages = useMemo(() => {
+    if (sample?.messages) {
+      return sample.messages;
+    } else if (runningSampleData) {
+      return messagesFromEvents(runningSampleData);
+    } else {
+      return [];
+    }
+  }, [sample?.messages, runningSampleData]);
 
   // Tab selection
   const onSelectedTab = (e: MouseEvent<HTMLElement>) => {
@@ -68,12 +92,15 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
     return false;
   };
 
-  const scorerNames = Object.keys(sample.scores || {});
   const sampleMetadatas = metadataViewsForSample(`${baseId}-${id}`, sample);
 
   const tabsetId = `task-sample-details-tab-${id}`;
   const targetId = `${tabsetId}-content`;
 
+  const handlePrintClick = useCallback(() => {
+    printSample(id, targetId);
+  }, [printSample, id, targetId]);
+
   const tools = [];
   if (!isVscode()) {
     tools.push(
@@ -81,51 +108,48 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
         key="sample-print-tool"
         label="Print"
         icon={ApplicationIcons.copy}
-        onClick={() => {
-          printSample(id, targetId);
-        }}
+        onClick={handlePrintClick}
       />,
     );
   }
 
+  // Is the sample running?
+  const running = isRunning(sampleSummary, runningSampleData);
+
   return (
     <Fragment>
-      <SampleSummaryView
-        parent_id={id}
-        sample={sample}
-        sampleDescriptor={sampleDescriptor}
-      />
+      {sample || sampleSummary ? (
+        <SampleSummaryView parent_id={id} sample={sample || sampleSummary} />
+      ) : undefined}
       <TabSet
         id={tabsetId}
         tabControlsClassName={clsx("text-size-base")}
         tabPanelsClassName={clsx(styles.tabPanel)}
         tools={tools}
       >
-        {sample.events && sample.events.length > 0 ? (
-          <TabPanel
-            key={kSampleTranscriptTabId}
-            id={kSampleTranscriptTabId}
-            className="sample-tab"
-            title="Transcript"
-            onSelected={onSelectedTab}
-            selected={
-              selectedTab === kSampleTranscriptTabId ||
-              selectedTab === undefined
-            }
-            scrollable={false}
-          >
-            <SampleTranscript
-              key={`${baseId}-transcript-display-${id}`}
-              id={`${baseId}-transcript-display-${id}`}
-              evalEvents={sample.events}
-              scrollRef={scrollRef}
-            />
-          </TabPanel>
-        ) : null}
+        <TabPanel
+          key={kSampleTranscriptTabId}
+          id={kSampleTranscriptTabId}
+          className="sample-tab"
+          title="Transcript"
+          onSelected={onSelectedTab}
+          selected={
+            selectedTab === kSampleTranscriptTabId || selectedTab === undefined
+          }
+          scrollable={false}
+        >
+          <TranscriptVirtualList
+            key={`${baseId}-transcript-display-${id}`}
+            id={`${baseId}-transcript-display-${id}`}
+            events={sampleEvents || []}
+            running={running}
+            scrollRef={scrollRef}
+          />
+        </TabPanel>
         <TabPanel
           key={kSampleMessagesTabId}
           id={kSampleMessagesTabId}
-          className={clsx("sample-tab", styles.fullWidth)}
+          className={clsx("sample-tab", styles.fullWidth, styles.chat)}
           title="Messages"
           onSelected={onSelectedTab}
           selected={selectedTab === kSampleMessagesTabId}
@@ -134,62 +158,37 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
           <ChatViewVirtualList
             key={`${baseId}-chat-${id}`}
             id={`${baseId}-chat-${id}`}
-            messages={sample.messages}
+            messages={sampleMessages}
             indented={true}
             scrollRef={scrollRef}
             toolCallStyle="complete"
+            running={running}
           />
         </TabPanel>
-        {scorerNames.length === 1 ? (
-          <TabPanel
-            key={kSampleScoringTabId}
-            id={kSampleScoringTabId}
-            className="sample-tab"
-            title="Scoring"
-            onSelected={onSelectedTab}
-            selected={selectedTab === kSampleScoringTabId}
-          >
-            <SampleScoreView
-              sample={sample}
-              sampleDescriptor={sampleDescriptor}
-              scorer={scorerNames[0]}
-            />
-          </TabPanel>
-        ) : (
-          <>
-            {Object.keys(sample.scores || {}).map((scorer) => {
-              const tabId = `score-${scorer}`;
-              return (
-                <TabPanel
-                  key={tabId}
-                  id={tabId}
-                  className="sample-tab"
-                  title={scorer}
-                  onSelected={onSelectedTab}
-                  selected={selectedTab === tabId}
-                >
-                  <SampleScoreView
-                    sample={sample}
-                    sampleDescriptor={sampleDescriptor}
-                    scorer={scorer}
-                  />
-                </TabPanel>
-              );
-            })}
-          </>
-        )}
-        {sampleMetadatas.length > 0 ? (
-          <TabPanel
-            id={kSampleMetdataTabId}
-            className={clsx("sample-tab")}
-            title="Metadata"
-            onSelected={onSelectedTab}
-            selected={selectedTab === kSampleMetdataTabId}
-          >
+        <TabPanel
+          key={kSampleScoringTabId}
+          id={kSampleScoringTabId}
+          className="sample-tab"
+          title="Scoring"
+          onSelected={onSelectedTab}
+          selected={selectedTab === kSampleScoringTabId}
+        >
+          <SampleScoresView sample={sample} />
+        </TabPanel>
+        <TabPanel
+          id={kSampleMetdataTabId}
+          className={clsx("sample-tab")}
+          title="Metadata"
+          onSelected={onSelectedTab}
+          selected={selectedTab === kSampleMetdataTabId}
+        >
+          {sampleMetadatas.length > 0 ? (
             <div className={clsx(styles.metadataPanel)}>{sampleMetadatas}</div>
-          </TabPanel>
-        ) : null}
-        {sample.error ? (
+          ) : (
+            <NoContentsPanel text="No metadata" />
+          )}
+        </TabPanel>
+        {sample?.error ? (
           <TabPanel
             id={kSampleErrorTabId}
             className="sample-tab"
@@ -205,14 +204,18 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
             </div>
           </TabPanel>
         ) : null}
-        {sample.messages.length < 100 ? (
-          <TabPanel
-            id={kSampleJsonTabId}
-            className={"sample-tab"}
-            title="JSON"
-            onSelected={onSelectedTab}
-            selected={selectedTab === kSampleJsonTabId}
-          >
+        <TabPanel
+          id={kSampleJsonTabId}
+          className={"sample-tab"}
+          title="JSON"
+          onSelected={onSelectedTab}
+          selected={selectedTab === kSampleJsonTabId}
+        >
+          {!sample ? (
+            <NoContentsPanel text="JSON not available" />
+          ) : sample.messages.length > 100 ? (
+            <NoContentsPanel text="JSON too large too display" />
+          ) : (
             <div className={clsx(styles.padded, styles.fullWidth)}>
               <JSONPanel
                 data={sample}
@@ -220,14 +223,17 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
                 className={clsx("text-size-small")}
               />
             </div>
-          </TabPanel>
-        ) : null}
+          )}
+        </TabPanel>
       </TabSet>
     </Fragment>
   );
 };
 
-const metadataViewsForSample = (id: string, sample: EvalSample) => {
+const metadataViewsForSample = (id: string, sample?: EvalSample) => {
+  if (!sample) {
+    return [];
+  }
   const sampleMetadatas = [];
 
   if (sample.model_usage && Object.keys(sample.model_usage).length > 0) {
@@ -360,3 +366,28 @@ const printSample = (id: string, targetId: string) => {
     }
   }
 };
+
+const isRunning = (
+  sampleSummary?: SampleSummary,
+  runningSampleData?: Events,
+): boolean => {
+  if (sampleSummary && sampleSummary.completed === false) {
+    // An explicitly incomplete sample summary
+    return true;
+  }
+
+  if (
+    !sampleSummary &&
+    (!runningSampleData || runningSampleData.length === 0)
+  ) {
+    // No sample summary yet and no running samples, must've just started
+    return true;
+  }
+
+  if (runningSampleData && runningSampleData.length > 0) {
+    // There are running samples
+    return true;
+  }
+
+  return false;
+};

inspect_ai/_view/www/src/samples/SampleSummaryView.module.css

@@ -19,6 +19,11 @@
   justify-content: center;
 }
 
+.centerValue {
+  display: flex;
+  align-items: center;
+}
+
 .wrap {
   word-wrap: anywhere;
 }
@@ -26,3 +31,8 @@
 .titled:hover {
   cursor: pointer;
 }
+
+.value {
+  flex-direction: column;
+  padding-top: 0.1em;
+}