inspect-ai 0.3.82__py3-none-any.whl → 0.3.84__py3-none-any.whl

inspect_ai/_view/www/src/samples/scores/SampleScoresGrid.module.css

@@ -0,0 +1,38 @@
+.container {
+  display: grid;
+  grid-template-columns:
+    minmax(auto, 1fr) minmax(auto, 1fr) minmax(auto, 1fr)
+    2fr;
+  column-gap: 0.75em;
+}
+
+.container .cell {
+  margin-bottom: 0.5em;
+}
+
+.fullWidth {
+  grid-column: 1 / -1;
+}
+
+.heading {
+  font-weight: 600;
+}
+
+.padded {
+  padding-bottom: 3em;
+}
+
+.separator {
+  height: 1px;
+  background-color: var(--bs-light-border-subtle);
+}
+
+.separatorPadded {
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}
+
+.headerSep {
+  margin-top: 0.1em;
+  margin-bottom: 0.2em;
+}

inspect_ai/_view/www/src/samples/scores/SampleScoresGrid.tsx

@@ -0,0 +1,118 @@
+import clsx from "clsx";
+import { FC } from "react";
+import { SampleSummary } from "../../api/types";
+import { EmptyPanel } from "../../components/EmptyPanel";
+import { MetaDataGrid } from "../../metadata/MetaDataGrid";
+import { useEvalDescriptor } from "../../state/hooks";
+import { EvalSample } from "../../types/log";
+import { SampleScores } from "./SampleScores";
+import styles from "./SampleScoresGrid.module.css";
+
+interface SampleScoresGridProps {
+  evalSample: EvalSample;
+  className?: string | string[];
+}
+
+export const SampleScoresGrid: FC<SampleScoresGridProps> = ({
+  evalSample,
+  className,
+}) => {
+  const evalDescriptor = useEvalDescriptor();
+  if (!evalDescriptor) {
+    return <EmptyPanel>No Sample Selected</EmptyPanel>;
+  }
+  return (
+    <div className={clsx(className, styles.container)}>
+      <div
+        className={clsx(
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+        )}
+      >
+        Scorer
+      </div>
+      <div
+        className={clsx(
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+        )}
+      >
+        Answer
+      </div>
+      <div
+        className={clsx(
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+        )}
+      >
+        Score
+      </div>
+      <div
+        className={clsx(
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+        )}
+      >
+        Explanation
+      </div>
+      <div
+        className={clsx(styles.separator, styles.fullWidth, styles.headerSep)}
+      ></div>
+
+      {Object.keys(evalSample.scores || {}).map((scorer) => {
+        if (!evalSample.scores) {
+          return undefined;
+        }
+        const scoreData = evalSample.scores[scorer];
+        const explanation = scoreData.explanation || "(No Explanation)";
+        const answer = scoreData.answer;
+        let metadata = scoreData.metadata || {};
+
+        return (
+          <>
+            <div className={clsx("text-size-base", styles.cell)}>{scorer}</div>
+            <div className={clsx(styles.cell, "text-size-base")}>{answer}</div>
+            <div className={clsx(styles.cell, "text-size-base")}>
+              <SampleScores
+                sample={evalSample as any as SampleSummary}
+                scorer={scorer}
+              />
+            </div>
+            <div className={clsx("text-size-base", styles.cell)}>
+              {explanation}
+            </div>
+
+            {Object.keys(metadata).length > 0 ? (
+              <>
+                <div
+                  className={clsx(
+                    "text-size-smaller",
+                    "text-style-label",
+                    "text-style-secondary",
+                    styles.fullWidth,
+                  )}
+                >
+                  Metadata
+                </div>
+                <div className={clsx(styles.fullWidth)}>
+                  <MetaDataGrid entries={metadata} />
+                </div>
+                <div
+                  className={clsx(
+                    styles.separator,
+                    styles.separatorPadded,
+                    styles.fullWidth,
+                  )}
+                ></div>
+              </>
+            ) : undefined}
+          </>
+        );
+      })}
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/scores/{SampleScoreView.module.css → SampleScoresView.module.css}

@@ -1,6 +1,7 @@
 .container {
   margin-top: 0.5em;
   padding-left: 0;
+  padding-right: 0;
 }
 
 .label {
@@ -12,7 +13,7 @@
 }
 
 .wordBreak {
-  word-break: break-all;
+  word-break: break-word;
 }
 
 .scoreTable {
@@ -51,3 +52,11 @@
 .noTop {
   margin-top: 0 !important;
 }
+
+.scoreCard {
+  padding-top: 0.5em;
+}
+
+.scores {
+  padding-top: 1em;
+}

inspect_ai/_view/www/src/samples/scores/SampleScoresView.tsx

@@ -0,0 +1,78 @@
+import clsx from "clsx";
+import { Card, CardBody } from "../../components/Card";
+import { MarkdownDiv } from "../../components/MarkdownDiv";
+import { EvalSample } from "../../types/log";
+import { inputString } from "../../utils/format";
+
+import { FC } from "react";
+import ExpandablePanel from "../../components/ExpandablePanel";
+import { useEvalDescriptor } from "../../state/hooks";
+import { SampleScoresGrid } from "./SampleScoresGrid";
+import styles from "./SampleScoresView.module.css";
+
+interface SampleScoresViewProps {
+  sample?: EvalSample;
+  className?: string | string[];
+}
+
+export const SampleScoresView: FC<SampleScoresViewProps> = ({
+  sample,
+  className,
+}) => {
+  const evalDescriptor = useEvalDescriptor();
+  if (!evalDescriptor) {
+    return undefined;
+  }
+  if (!sample) {
+    return undefined;
+  }
+
+  const scoreInput = inputString(sample.input);
+  if (sample.choices && sample.choices.length > 0) {
+    scoreInput.push("");
+    scoreInput.push(
+      ...sample.choices.map((choice, index) => {
+        return `${String.fromCharCode(65 + index)}) ${choice}`;
+      }),
+    );
+  }
+
+  return (
+    <div
+      className={clsx(
+        "container-fluid",
+        className,
+        "font-size-base",
+        styles.container,
+      )}
+    >
+      <Card className={clsx(styles.scoreCard)}>
+        <CardBody>
+          <div
+            className={clsx(
+              "text-size-small",
+              "text-style-label",
+              "text-style-secondary",
+            )}
+          >
+            Input
+          </div>
+          <ExpandablePanel
+            lines={10}
+            id={`sample-score-${sample.id}-${sample.epoch}`}
+            collapse={true}
+          >
+            <MarkdownDiv
+              markdown={scoreInput.join("\n")}
+              className={clsx(styles.wordBreak, "text-size-base")}
+            />
+          </ExpandablePanel>
+          <SampleScoresGrid
+            evalSample={sample}
+            className={clsx(styles.scores)}
+          />
+        </CardBody>
+      </Card>
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx

@@ -1,6 +1,6 @@
 import { FC } from "react";
 import { ApplicationIcons } from "../../appearance/icons";
-import { SampleLimitEvent, Type10 } from "../../types/log";
+import { SampleLimitEvent, Type9 } from "../../types/log";
 import { EventPanel } from "./event/EventPanel";
 
 interface SampleLimitEventViewProps {
@@ -17,7 +17,7 @@ export const SampleLimitEventView: FC<SampleLimitEventViewProps> = ({
   event,
   className,
 }) => {
-  const resolve_title = (type: Type10) => {
+  const resolve_title = (type: Type9) => {
     switch (type) {
       case "custom":
         return "Custom Limit Exceeded";
@@ -34,7 +34,7 @@ export const SampleLimitEventView: FC<SampleLimitEventViewProps> = ({
     }
   };
 
-  const resolve_icon = (type: Type10) => {
+  const resolve_icon = (type: Type9) => {
     switch (type) {
       case "custom":
         return ApplicationIcons.limits.custom;

inspect_ai/_view/www/src/samples/transcript/ToolEventView.tsx

@@ -9,6 +9,7 @@ import { TranscriptView } from "./TranscriptView";
 import clsx from "clsx";
 import { FC, useMemo } from "react";
 import { PulsingDots } from "../../components/PulsingDots";
+import { ChatView } from "../chat/ChatView";
 import { formatTiming, formatTitle } from "./event/utils";
 import styles from "./ToolEventView.module.css";
 
@@ -34,10 +35,19 @@ export const ToolEventView: FC<ToolEventViewProps> = ({
     [event.function, event.arguments],
   );
 
-  // Find an approval if there is one
-  const approvalEvent = event.events.find((e) => {
-    return e.event === "approval";
-  });
+  const { approvalEvent, lastModelEvent } = useMemo(() => {
+    // Find an approval if there is one
+    const approvalEvent = event.events.find((e) => {
+      return e.event === "approval";
+    });
+
+    // Find a model message to render, if there is one
+    const lastModelEvent = [...event.events].reverse().find((e) => {
+      return e.event === "model";
+    });
+
+    return { approvalEvent, lastModelEvent };
+  }, [event.events]);
 
   const title = `Tool: ${event.view?.title || event.function}`;
   return (
@@ -58,6 +68,16 @@ export const ToolEventView: FC<ToolEventViewProps> = ({
           mode="compact"
           view={event.view ? event.view : undefined}
         />
+
+        {lastModelEvent && lastModelEvent.event === "model" ? (
+          <ChatView
+            id={`${id}-toolcall-chatmessage`}
+            messages={lastModelEvent.output.choices.map((m) => m.message)}
+            numbered={false}
+            toolCallStyle="compact"
+          />
+        ) : undefined}
+
         {approvalEvent ? (
           <ApprovalEventView
             event={approvalEvent}
@@ -76,6 +96,7 @@ export const ToolEventView: FC<ToolEventViewProps> = ({
         <TranscriptView
           id={`${id}-subtask`}
           data-name="Transcript"
+          data-default={event.failed || event.agent ? true : null}
           events={event.events}
           depth={depth + 1}
         />

inspect_ai/_view/www/src/samples/transcript/event/EventPanel.tsx

@@ -1,5 +1,11 @@
 import clsx from "clsx";
-import { FC, isValidElement, ReactNode, useCallback } from "react";
+import {
+  FC,
+  isValidElement,
+  ReactElement,
+  ReactNode,
+  useCallback,
+} from "react";
 import { ApplicationIcons } from "../../../appearance/icons";
 import { EventNavs } from "./EventNavs";
 
@@ -49,7 +55,11 @@ export const EventPanel: FC<EventPanelProps> = ({
   const filteredArrChildren = (
     Array.isArray(children) ? children : [children]
   ).filter((child) => !!child);
-  const defaultPillId = pillId(0);
+
+  const defaultPill = filteredArrChildren.findIndex((node) => {
+    return hasDataDefault(node) && node.props["data-default"];
+  });
+  const defaultPillId = defaultPill !== -1 ? pillId(defaultPill) : pillId(0);
 
   const [selectedNav, setSelectedNav] = useProperty(id, "selectedNav", {
     defaultValue: defaultPillId,
@@ -186,3 +196,20 @@ export const EventPanel: FC<EventPanelProps> = ({
   );
   return card;
 };
+
+// Typeguard for reading default value from pills
+interface DataDefaultProps {
+  "data-default"?: boolean;
+  [key: string]: any;
+}
+
+function hasDataDefault(
+  node: ReactNode,
+): node is ReactElement<DataDefaultProps> {
+  return (
+    isValidElement(node) &&
+    node.props !== null &&
+    typeof node.props === "object" &&
+    "data-default" in node.props
+  );
+}

inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.tsx

@@ -216,7 +216,6 @@ const createMessageRenderer = (name: string, role: string): ChangeType => {
   return {
     type: name,
     match: (changes: JsonChange[]) => {
-      console.log(changes);
       if (changes.length === 1) {
         const change = changes[0];
         if (change.op === "add" && change.path.match(/\/messages\/\d+/)) {

inspect_ai/_view/www/src/state/hooks.ts

@@ -78,7 +78,9 @@ export const useScores = () => {
       return [];
     }
 
-    return getAvailableScorers(selectedLogSummary, sampleSummaries) || [];
+    const result =
+      getAvailableScorers(selectedLogSummary, sampleSummaries) || [];
+    return result;
   }, [selectedLogSummary, sampleSummaries]);
 };
 
@@ -361,8 +363,8 @@ export const usePrismHighlight = (toolCallContent?: string) => {
       toolCallContent.length <= kPrismRenderMaxSize
     ) {
       requestAnimationFrame(() => {
-        const codeBlocks = toolViewRef.current!.querySelectorAll("pre code");
-        codeBlocks.forEach((block) => {
+        const codeBlocks = toolViewRef.current?.querySelectorAll("pre code");
+        codeBlocks?.forEach((block) => {
           if (block.className.includes("language-")) {
             block.classList.add("sourceCode");
             highlightElement(block as HTMLElement);

inspect_ai/_view/www/src/state/logPolling.ts

@@ -133,7 +133,11 @@ export function createLogPolling(
           log.debug(`Stop polling running samples: ${logFileName}`);
 
           // Clear pending summaries and refresh in one transaction
-          if (loadedPendingSamples) {
+          if (
+            loadedPendingSamples ||
+            state.log.selectedLogSummary?.status === "started"
+          ) {
+            log.debug(`Refresh log: ${logFileName}`);
             await refreshLog(logFileName, true);
           }
 

inspect_ai/_view/www/src/state/logSlice.ts

@@ -45,6 +45,9 @@ export interface LogSlice {
 
     // Refresh the current log
     refreshLog: () => Promise<void>;
+
+    // Poll the currently selected log
+    pollLog: () => Promise<void>;
   };
 }
 
@@ -175,6 +178,13 @@ export const createLogSlice = (
         }
       },
 
+      pollLog: async () => {
+        const currentLog = get().log.loadedLog;
+        if (currentLog) {
+          logPolling.startPolling(currentLog);
+        }
+      },
+
       refreshLog: async () => {
         const state = get();
         const api = state.api;

inspect_ai/_view/www/src/state/samplePolling.ts

@@ -123,7 +123,10 @@ export function createSamplePolling(
 
         // Also fetch a fresh sample and clear the runnning Events
         // (if there were ever running events)
-        if (state.sample.runningEvents.length > 0) {
+        if (
+          state.sample.runningEvents.length > 0 ||
+          state.sample.sampleStatus === "streaming"
+        ) {
           try {
             log.debug(
               `LOADING COMPLETED SAMPLE AFTER FLUSH: ${summary.id}-${summary.epoch}`,

inspect_ai/_view/www/src/state/sampleSlice.ts

@@ -23,6 +23,11 @@ export interface SampleSlice {
       logFile: string,
       sampleSummary: SampleSummary,
     ) => Promise<void>;
+
+    pollSample: (
+      logFile: string,
+      sampleSummary: SampleSummary,
+    ) => Promise<void>;
   };
 }
 
@@ -68,6 +73,14 @@ export const createSampleSlice = (
         set((state) => {
           state.sample.sampleError = error;
         }),
+      pollSample: async (logFile: string, sampleSummary: SampleSummary) => {
+        // Poll running sample
+        const state = get();
+        if (state.log.loadedLog && state.sample.selectedSample) {
+          samplePolling.startPolling(logFile, sampleSummary);
+          state.sampleActions.setSampleStatus("streaming");
+        }
+      },
       loadSample: async (logFile: string, sampleSummary: SampleSummary) => {
         const sampleActions = get().sampleActions;