inspect-ai 0.3.80__py3-none-any.whl → 0.3.82__py3-none-any.whl

inspect_ai/_view/www/src/metadata/RenderedContent.tsx

@@ -7,7 +7,7 @@ import { formatNumber } from "../utils/format";
 import { MetaDataView } from "./MetaDataView";
 
 import clsx from "clsx";
-import React, { Fragment, JSX } from "react";
+import { FC, Fragment, isValidElement, JSX, ReactNode } from "react";
 import JSONPanel from "../components/JsonPanel";
 import { isJson } from "../utils/json";
 import styles from "./RenderedContent.module.css";
@@ -21,7 +21,7 @@ interface RenderedContentProps {
 /**
  * Renders content based on its type using registered content renderers.
  */
-export const RenderedContent: React.FC<RenderedContentProps> = ({
+export const RenderedContent: FC<RenderedContentProps> = ({
   id,
   entry,
 }): JSX.Element => {
@@ -43,8 +43,7 @@ export const RenderedContent: React.FC<RenderedContentProps> = ({
 
   if (renderer) {
     const { rendered } = renderer.render(id, entry);
-    // Check if rendered is already a valid ReactNode (JSX.Element)
-    if (rendered !== undefined && React.isValidElement(rendered)) {
+    if (rendered !== undefined && isValidElement(rendered)) {
       return rendered;
     }
   }
@@ -185,7 +184,7 @@ const contentRenderers: Record<string, ContentRenderer> = {
       return typeof entry.value === "object" && entry.name === "web_search";
     },
     render: (_id, entry) => {
-      const results: React.ReactNode[] = [];
+      const results: ReactNode[] = [];
       results.push(
         <div className={styles.query}>
           <i className={ApplicationIcons.search}></i> {entry.value.query}
@@ -253,7 +252,6 @@ const contentRenderers: Record<string, ContentRenderer> = {
       return typeof entry.value === "object";
     },
     render: (id, entry) => {
-      // Generate a json preview
       return {
         rendered: (
           <MetaDataView

inspect_ai/_view/www/src/plan/ScorerDetailView.tsx

@@ -17,7 +17,7 @@ export const ScorerDetailView: FC<ScorerDetailViewProps> = ({
 }) => {
   // Merge scores into params
   if (scores.length > 1) {
-    params["scores"] = scores;
+    params = { ...params, ["scores"]: scores };
   }
 
   return (

inspect_ai/_view/www/src/samples/InlineSampleDisplay.module.css

@@ -1,8 +1,16 @@
 .container {
-  flex-direction: row;
   width: 100%;
+  display: flex;
+  flex-direction: column;
 }
 
 .body {
   margin: 1em 1em 1em 1em;
 }
+
+.scroller {
+  overflow-y: auto;
+  width: 100%;
+  height: 100%;
+  position: relative;
+}

inspect_ai/_view/www/src/samples/InlineSampleDisplay.tsx

@@ -1,21 +1,17 @@
-import { FC, RefObject } from "react";
+import { FC, useEffect, useRef } from "react";
 import { ErrorPanel } from "../components/ErrorPanel";
-import { ProgressBar } from "../components/ProgressBar";
-import { EvalSample } from "../types/log";
 import { SampleDisplay } from "./SampleDisplay";
-import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
 
+import clsx from "clsx";
+import { ProgressBar } from "../components/ProgressBar";
+import { useLogSelection, usePrevious, useSampleData } from "../state/hooks";
+import { useStore } from "../state/store";
 import styles from "./InlineSampleDisplay.module.css";
 
 interface InlineSampleDisplayProps {
   id: string;
-  sampleStatus: string;
-  sampleError?: Error;
-  sample?: EvalSample;
-  sampleDescriptor: SamplesDescriptor;
   selectedTab?: string;
   setSelectedTab: (tab: string) => void;
-  scrollRef: RefObject<HTMLDivElement | null>;
 }
 
 /**
@@ -23,30 +19,73 @@ interface InlineSampleDisplayProps {
  */
 export const InlineSampleDisplay: FC<InlineSampleDisplayProps> = ({
   id,
-  sample,
-  sampleStatus,
-  sampleError,
-  sampleDescriptor,
   selectedTab,
   setSelectedTab,
-  scrollRef,
 }) => {
+  // Sample hooks
+  const sampleData = useSampleData();
+  const loadSample = useStore((state) => state.sampleActions.loadSample);
+  const logSelection = useLogSelection();
+
+  // Sample Loading
+  const prevCompleted = usePrevious(
+    logSelection.sample?.completed !== undefined
+      ? logSelection.sample.completed
+      : true,
+  );
+  const prevLogFile = usePrevious<string | undefined>(logSelection.logFile);
+  useEffect(() => {
+    if (logSelection.logFile && logSelection.sample) {
+      const currentSampleCompleted =
+        logSelection.sample?.completed !== undefined
+          ? logSelection.sample.completed
+          : true;
+
+      if (
+        prevLogFile !== logSelection.logFile ||
+        sampleData.sample?.id !== logSelection.sample.id ||
+        sampleData.sample?.epoch !== logSelection.sample.epoch ||
+        currentSampleCompleted !== prevCompleted
+      ) {
+        loadSample(logSelection.logFile, logSelection.sample);
+      }
+    }
+  }, [
+    logSelection.logFile,
+    logSelection.sample?.id,
+    logSelection.sample?.epoch,
+    logSelection.sample?.completed,
+    sampleData.sample?.id,
+    sampleData.sample?.epoch,
+  ]);
+
+  // Scroll ref
+  const scrollRef = useRef<HTMLDivElement>(null);
   return (
     <div className={styles.container}>
-      <ProgressBar animating={sampleStatus === "loading"} />
-      <div className={styles.body}>
-        {sampleError ? (
-          <ErrorPanel title="Unable to load sample" error={sampleError} />
-        ) : (
-          <SampleDisplay
-            id={id}
-            sample={sample}
-            sampleDescriptor={sampleDescriptor}
-            selectedTab={selectedTab}
-            setSelectedTab={setSelectedTab}
-            scrollRef={scrollRef}
-          />
-        )}
+      <ProgressBar
+        animating={
+          sampleData.status === "loading" || sampleData.status === "streaming"
+        }
+      />
+      <div className={clsx(styles.scroller)} ref={scrollRef}>
+        <div className={styles.body}>
+          {sampleData.error ? (
+            <ErrorPanel
+              title="Unable to load sample"
+              error={sampleData.error}
+            />
+          ) : (
+            <SampleDisplay
+              id={id}
+              sample={sampleData.sample}
+              runningEvents={sampleData.running}
+              selectedTab={selectedTab}
+              setSelectedTab={setSelectedTab}
+              scrollRef={scrollRef}
+            />
+          )}
+        </div>
       </div>
     </div>
   );

inspect_ai/_view/www/src/samples/SampleDialog.tsx

@@ -1,27 +1,22 @@
 import { ApplicationIcons } from "../appearance/icons";
 import { LargeModal, ModalTool, ModalTools } from "../components/LargeModal";
 
-import { FC, Ref, RefObject, useCallback, useMemo, useRef } from "react";
+import { FC, Ref, useCallback, useEffect, useMemo, useRef } from "react";
 import { ErrorPanel } from "../components/ErrorPanel";
-import { EvalSample } from "../types/log";
+import { useLogSelection, usePrevious, useSampleData } from "../state/hooks";
+import { useStatefulScrollPosition } from "../state/scrolling";
+import { useStore } from "../state/store";
 import { SampleDisplay } from "./SampleDisplay";
-import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
 
 interface SampleDialogProps {
   id: string;
   title: string;
-  sampleStatus: string;
-  sampleError?: Error;
-  sample?: EvalSample;
-  sampleDescriptor: SamplesDescriptor;
   selectedTab?: string;
   setSelectedTab: (tab: string) => void;
   showingSampleDialog: boolean;
   setShowingSampleDialog: (showing: boolean) => void;
   nextSample: () => void;
   prevSample: () => void;
-  sampleScrollPositionRef: RefObject<number>;
-  setSampleScrollPosition: (position: number) => void;
 }
 
 /**
@@ -30,21 +25,55 @@ interface SampleDialogProps {
 export const SampleDialog: FC<SampleDialogProps> = ({
   id,
   title,
-  sample,
-  sampleDescriptor,
   nextSample,
   prevSample,
-  sampleStatus,
-  sampleError,
   showingSampleDialog,
   setShowingSampleDialog,
   selectedTab,
   setSelectedTab,
-  sampleScrollPositionRef,
-  setSampleScrollPosition,
 }) => {
+  // Scroll referernce (attach stateful trackign)
   const scrollRef: Ref<HTMLDivElement> = useRef(null);
+  useStatefulScrollPosition(scrollRef, "sample-dialog");
 
+  // Sample hooks
+  const sampleData = useSampleData();
+  const loadSample = useStore((state) => state.sampleActions.loadSample);
+  const logSelection = useLogSelection();
+
+  // Load sample
+  const prevCompleted = usePrevious(
+    logSelection.sample?.completed !== undefined
+      ? logSelection.sample.completed
+      : true,
+  );
+  const prevLogFile = usePrevious<string | undefined>(logSelection.logFile);
+  useEffect(() => {
+    if (logSelection.logFile && logSelection.sample) {
+      const currentSampleCompleted =
+        logSelection.sample.completed !== undefined
+          ? logSelection.sample.completed
+          : true;
+
+      if (
+        prevLogFile !== logSelection.logFile ||
+        sampleData.sample?.id !== logSelection.sample.id ||
+        sampleData.sample?.epoch !== logSelection.sample.epoch ||
+        currentSampleCompleted !== prevCompleted
+      ) {
+        loadSample(logSelection.logFile, logSelection.sample);
+      }
+    }
+  }, [
+    logSelection.logFile,
+    logSelection.sample?.id,
+    logSelection.sample?.epoch,
+    logSelection.sample?.completed,
+    sampleData.sample?.id,
+    sampleData.sample?.epoch,
+  ]);
+
+  // Tools
   const tools = useMemo<ModalTools>(() => {
     const nextTool: ModalTool = {
       label: "Next Sample",
@@ -100,18 +129,18 @@ export const SampleDialog: FC<SampleDialogProps> = ({
       onkeyup={handleKeyUp}
       visible={showingSampleDialog}
       onHide={onHide}
-      showProgress={sampleStatus === "loading"}
-      initialScrollPositionRef={sampleScrollPositionRef}
-      setInitialScrollPosition={setSampleScrollPosition}
+      showProgress={
+        sampleData.status === "loading" || sampleData.status === "streaming"
+      }
       scrollRef={scrollRef}
     >
-      {sampleError ? (
-        <ErrorPanel title="Sample Error" error={sampleError} />
+      {sampleData.error ? (
+        <ErrorPanel title="Sample Error" error={sampleData.error} />
       ) : (
         <SampleDisplay
           id={id}
-          sample={sample}
-          sampleDescriptor={sampleDescriptor}
+          sample={sampleData.sample}
+          runningEvents={sampleData.running}
           selectedTab={selectedTab}
           setSelectedTab={setSelectedTab}
           scrollRef={scrollRef}

inspect_ai/_view/www/src/samples/SampleDisplay.module.css

@@ -35,3 +35,7 @@
   column-gap: 0.5em;
   min-width: 200px;
 }
+
+.chat {
+  padding-top: 1em;
+}

inspect_ai/_view/www/src/samples/SampleDisplay.tsx

@@ -10,10 +10,18 @@ import { ToolButton } from "../components/ToolButton";
 import { SampleScoreView } from "./scores/SampleScoreView";
 
 import clsx from "clsx";
-import { FC, Fragment, MouseEvent, RefObject } from "react";
+import {
+  FC,
+  Fragment,
+  MouseEvent,
+  RefObject,
+  useCallback,
+  useMemo,
+} from "react";
+import { SampleSummary } from "../api/types";
 import { Card, CardBody, CardHeader } from "../components/Card";
-import { EmptyPanel } from "../components/EmptyPanel";
 import { JSONPanel } from "../components/JsonPanel";
+import { NoContentsPanel } from "../components/NoContentsPanel";
 import {
   kSampleErrorTabId,
   kSampleJsonTabId,
@@ -22,23 +30,25 @@ import {
   kSampleScoringTabId,
   kSampleTranscriptTabId,
 } from "../constants";
-import { EvalSample } from "../types/log";
+import { useSampleSummaries } from "../state/hooks";
+import { useStore } from "../state/store";
+import { EvalSample, Events } from "../types/log";
 import { ModelTokenTable } from "../usage/ModelTokenTable";
 import { formatTime } from "../utils/format";
 import { printHeadingHtml, printHtml } from "../utils/print";
 import { ChatViewVirtualList } from "./chat/ChatViewVirtualList";
-import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
+import { messagesFromEvents } from "./chat/messages";
 import styles from "./SampleDisplay.module.css";
 import { SampleSummaryView } from "./SampleSummaryView";
-import { SampleTranscript } from "./transcript/SampleTranscript";
+import { TranscriptVirtualList } from "./transcript/TranscriptView";
 
 interface SampleDisplayProps {
   id: string;
   sample?: EvalSample;
-  sampleDescriptor: SamplesDescriptor;
   selectedTab?: string;
   setSelectedTab: (tab: string) => void;
   scrollRef: RefObject<HTMLDivElement | null>;
+  runningEvents?: Events;
 }
 
 /**
@@ -47,18 +57,32 @@ interface SampleDisplayProps {
 export const SampleDisplay: FC<SampleDisplayProps> = ({
   id,
   sample,
-  sampleDescriptor,
   selectedTab,
   setSelectedTab,
   scrollRef,
+  runningEvents: runningSampleData,
 }) => {
   // Tab ids
   const baseId = `sample-dialog`;
+  const sampleSummaries = useSampleSummaries();
+  const selectedSampleIndex = useStore(
+    (state) => state.log.selectedSampleIndex,
+  );
 
-  if (!sample) {
-    // Placeholder
-    return <EmptyPanel />;
-  }
+  const sampleSummary = sampleSummaries[selectedSampleIndex];
+
+  // Consolidate the events and messages into the proper list
+  // whether running or not
+  const sampleEvents = sample?.events || runningSampleData;
+  const sampleMessages = useMemo(() => {
+    if (sample?.messages) {
+      return sample.messages;
+    } else if (runningSampleData) {
+      return messagesFromEvents(runningSampleData);
+    } else {
+      return [];
+    }
+  }, [sample?.messages, runningSampleData]);
 
   // Tab selection
   const onSelectedTab = (e: MouseEvent<HTMLElement>) => {
@@ -68,12 +92,16 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
     return false;
   };
 
-  const scorerNames = Object.keys(sample.scores || {});
+  const scorerNames = Object.keys(sample?.scores || {});
   const sampleMetadatas = metadataViewsForSample(`${baseId}-${id}`, sample);
 
   const tabsetId = `task-sample-details-tab-${id}`;
   const targetId = `${tabsetId}-content`;
 
+  const handlePrintClick = useCallback(() => {
+    printSample(id, targetId);
+  }, [printSample, id, targetId]);
+
   const tools = [];
   if (!isVscode()) {
     tools.push(
@@ -81,51 +109,48 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
         key="sample-print-tool"
         label="Print"
         icon={ApplicationIcons.copy}
-        onClick={() => {
-          printSample(id, targetId);
-        }}
+        onClick={handlePrintClick}
       />,
     );
   }
 
+  // Is the sample running?
+  const running = isRunning(sampleSummary, runningSampleData);
+
   return (
     <Fragment>
-      <SampleSummaryView
-        parent_id={id}
-        sample={sample}
-        sampleDescriptor={sampleDescriptor}
-      />
+      {sample || sampleSummary ? (
+        <SampleSummaryView parent_id={id} sample={sample || sampleSummary} />
+      ) : undefined}
       <TabSet
         id={tabsetId}
         tabControlsClassName={clsx("text-size-base")}
         tabPanelsClassName={clsx(styles.tabPanel)}
         tools={tools}
       >
-        {sample.events && sample.events.length > 0 ? (
-          <TabPanel
-            key={kSampleTranscriptTabId}
-            id={kSampleTranscriptTabId}
-            className="sample-tab"
-            title="Transcript"
-            onSelected={onSelectedTab}
-            selected={
-              selectedTab === kSampleTranscriptTabId ||
-              selectedTab === undefined
-            }
-            scrollable={false}
-          >
-            <SampleTranscript
-              key={`${baseId}-transcript-display-${id}`}
-              id={`${baseId}-transcript-display-${id}`}
-              evalEvents={sample.events}
-              scrollRef={scrollRef}
-            />
-          </TabPanel>
-        ) : null}
+        <TabPanel
+          key={kSampleTranscriptTabId}
+          id={kSampleTranscriptTabId}
+          className="sample-tab"
+          title="Transcript"
+          onSelected={onSelectedTab}
+          selected={
+            selectedTab === kSampleTranscriptTabId || selectedTab === undefined
+          }
+          scrollable={false}
+        >
+          <TranscriptVirtualList
+            key={`${baseId}-transcript-display-${id}`}
+            id={`${baseId}-transcript-display-${id}`}
+            events={sampleEvents || []}
+            running={running}
+            scrollRef={scrollRef}
+          />
+        </TabPanel>
         <TabPanel
           key={kSampleMessagesTabId}
           id={kSampleMessagesTabId}
-          className={clsx("sample-tab", styles.fullWidth)}
+          className={clsx("sample-tab", styles.fullWidth, styles.chat)}
           title="Messages"
           onSelected={onSelectedTab}
           selected={selectedTab === kSampleMessagesTabId}
@@ -134,13 +159,14 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
           <ChatViewVirtualList
             key={`${baseId}-chat-${id}`}
             id={`${baseId}-chat-${id}`}
-            messages={sample.messages}
+            messages={sampleMessages}
             indented={true}
             scrollRef={scrollRef}
             toolCallStyle="complete"
+            running={running}
           />
         </TabPanel>
-        {scorerNames.length === 1 ? (
+        {sample && scorerNames.length === 1 ? (
           <TabPanel
             key={kSampleScoringTabId}
             id={kSampleScoringTabId}
@@ -149,47 +175,43 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
             onSelected={onSelectedTab}
             selected={selectedTab === kSampleScoringTabId}
           >
-            <SampleScoreView
-              sample={sample}
-              sampleDescriptor={sampleDescriptor}
-              scorer={scorerNames[0]}
-            />
+            <SampleScoreView sample={sample} scorer={scorerNames[0]} />
           </TabPanel>
         ) : (
           <>
-            {Object.keys(sample.scores || {}).map((scorer) => {
-              const tabId = `score-${scorer}`;
-              return (
-                <TabPanel
-                  key={tabId}
-                  id={tabId}
-                  className="sample-tab"
-                  title={scorer}
-                  onSelected={onSelectedTab}
-                  selected={selectedTab === tabId}
-                >
-                  <SampleScoreView
-                    sample={sample}
-                    sampleDescriptor={sampleDescriptor}
-                    scorer={scorer}
-                  />
-                </TabPanel>
-              );
-            })}
+            {sample
+              ? Object.keys(sample?.scores || {}).map((scorer) => {
+                  const tabId = `score-${scorer}`;
+                  return (
+                    <TabPanel
+                      key={tabId}
+                      id={tabId}
+                      className="sample-tab"
+                      title={scorer}
+                      onSelected={onSelectedTab}
+                      selected={selectedTab === tabId}
+                    >
+                      <SampleScoreView sample={sample} scorer={scorer} />
+                    </TabPanel>
+                  );
+                })
+              : undefined}
           </>
         )}
-        {sampleMetadatas.length > 0 ? (
-          <TabPanel
-            id={kSampleMetdataTabId}
-            className={clsx("sample-tab")}
-            title="Metadata"
-            onSelected={onSelectedTab}
-            selected={selectedTab === kSampleMetdataTabId}
-          >
+        <TabPanel
+          id={kSampleMetdataTabId}
+          className={clsx("sample-tab")}
+          title="Metadata"
+          onSelected={onSelectedTab}
+          selected={selectedTab === kSampleMetdataTabId}
+        >
+          {sampleMetadatas.length > 0 ? (
             <div className={clsx(styles.metadataPanel)}>{sampleMetadatas}</div>
-          </TabPanel>
-        ) : null}
-        {sample.error ? (
+          ) : (
+            <NoContentsPanel text="No metadata" />
+          )}
+        </TabPanel>
+        {sample?.error ? (
           <TabPanel
             id={kSampleErrorTabId}
             className="sample-tab"
@@ -205,14 +227,18 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
             </div>
           </TabPanel>
         ) : null}
-        {sample.messages.length < 100 ? (
-          <TabPanel
-            id={kSampleJsonTabId}
-            className={"sample-tab"}
-            title="JSON"
-            onSelected={onSelectedTab}
-            selected={selectedTab === kSampleJsonTabId}
-          >
+        <TabPanel
+          id={kSampleJsonTabId}
+          className={"sample-tab"}
+          title="JSON"
+          onSelected={onSelectedTab}
+          selected={selectedTab === kSampleJsonTabId}
+        >
+          {!sample ? (
+            <NoContentsPanel text="JSON not available" />
+          ) : sample.messages.length > 100 ? (
+            <NoContentsPanel text="JSON too large too display" />
+          ) : (
             <div className={clsx(styles.padded, styles.fullWidth)}>
               <JSONPanel
                 data={sample}
@@ -220,14 +246,17 @@ export const SampleDisplay: FC<SampleDisplayProps> = ({
                 className={clsx("text-size-small")}
               />
             </div>
-          </TabPanel>
-        ) : null}
+          )}
+        </TabPanel>
       </TabSet>
     </Fragment>
   );
 };
 
-const metadataViewsForSample = (id: string, sample: EvalSample) => {
+const metadataViewsForSample = (id: string, sample?: EvalSample) => {
+  if (!sample) {
+    return [];
+  }
   const sampleMetadatas = [];
 
   if (sample.model_usage && Object.keys(sample.model_usage).length > 0) {
@@ -360,3 +389,28 @@ const printSample = (id: string, targetId: string) => {
     }
   }
 };
+
+const isRunning = (
+  sampleSummary?: SampleSummary,
+  runningSampleData?: Events,
+): boolean => {
+  if (sampleSummary && sampleSummary.completed === false) {
+    // An explicitly incomplete sample summary
+    return true;
+  }
+
+  if (
+    !sampleSummary &&
+    (!runningSampleData || runningSampleData.length === 0)
+  ) {
+    // No sample summary yet and no running samples, must've just started
+    return true;
+  }
+
+  if (runningSampleData && runningSampleData.length > 0) {
+    // There are running samples
+    return true;
+  }
+
+  return false;
+};