inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.tsx

@@ -16,6 +16,8 @@ import clsx from "clsx";
 import { EditorView, minimalSetup } from "codemirror";
 import { FC, useEffect, useMemo, useRef, useState } from "react";
 
+import { SampleSummary } from "../../../api/types";
+import { useEvalDescriptor } from "../../../state/hooks";
 import { ScoreFilter } from "../../../types";
 import { EvalDescriptor } from "../../descriptor/types";
 import { FilterError, filterSamples, scoreFilterItems } from "../filters";
@@ -30,7 +32,7 @@ interface FilteringResult {
 }
 
 interface SampleFilterProps {
-  evalDescriptor: EvalDescriptor;
+  samples: SampleSummary[];
   scoreFilter: ScoreFilter;
   setScoreFilter: (filter: ScoreFilter) => void;
 }
@@ -105,11 +107,12 @@ const editorTheme = EditorView.theme({
 // Helper functions
 const getFilteringResult = (
   evalDescriptor: EvalDescriptor,
+  sampleSummaries: SampleSummary[],
   filterValue: string,
 ): FilteringResult => {
   const { result, error } = filterSamples(
     evalDescriptor,
-    evalDescriptor.samples,
+    sampleSummaries,
     filterValue,
   );
   return { numSamples: result.length, error };
@@ -151,7 +154,7 @@ const getLints = (
 
 // Main component
 export const SampleFilter: FC<SampleFilterProps> = ({
-  evalDescriptor,
+  samples,
   scoreFilter,
   setScoreFilter,
 }) => {
@@ -160,9 +163,10 @@ export const SampleFilter: FC<SampleFilterProps> = ({
   const linterCompartment = useRef<Compartment>(new Compartment());
   const autocompletionCompartment = useRef<Compartment>(new Compartment());
   const updateListenerCompartment = useRef<Compartment>(new Compartment());
+  const evalDescriptor = useEvalDescriptor();
 
   const filterItems = useMemo(
-    () => scoreFilterItems(evalDescriptor),
+    () => (evalDescriptor ? scoreFilterItems(evalDescriptor) : []),
     [evalDescriptor],
   );
 
@@ -186,9 +190,13 @@ export const SampleFilter: FC<SampleFilterProps> = ({
 
   const makeUpdateListener = () =>
     EditorView.updateListener.of((update) => {
-      if (update.docChanged) {
+      if (update.docChanged && evalDescriptor) {
         const newValue = update.state.doc.toString();
-        const filteringResult = getFilteringResult(evalDescriptor, newValue);
+        const filteringResult = getFilteringResult(
+          evalDescriptor,
+          samples,
+          newValue,
+        );
         if (!filteringResult.error) {
           setScoreFilter({ value: newValue });
         }
@@ -229,9 +237,11 @@ export const SampleFilter: FC<SampleFilterProps> = ({
     const currentValue = editorViewRef.current.state.doc.toString();
     if (scoreFilter.value === currentValue) return;
 
-    setFilteringResultInstant(
-      getFilteringResult(evalDescriptor, scoreFilter.value || ""),
-    );
+    if (evalDescriptor) {
+      setFilteringResultInstant(
+        getFilteringResult(evalDescriptor, samples, scoreFilter.value || ""),
+      );
+    }
     editorViewRef.current.dispatch({
       changes: {
         from: 0,

inspect_ai/_view/www/src/samples/sampleDataAdapter.ts

@@ -0,0 +1,33 @@
+import { EventData, SampleData } from "../api/types";
+import { Events } from "../types/log";
+import { resolveAttachments } from "../utils/attachments";
+
+export const sampleDataAdapter = () => {
+  const attachments: Record<string, string> = {};
+  const events: Record<string, EventData> = {};
+
+  return {
+    addData: (data: SampleData) => {
+      data.attachments.forEach((a) => {
+        if (attachments[a.hash] === undefined) {
+          attachments[a.hash] = a.content;
+        }
+      });
+
+      data.events.forEach((e) => {
+        if (events[e.event_id] === undefined) {
+          events[e.event_id] = e;
+        }
+      });
+    },
+    resolvedEvents: (): Events => {
+      const eventDatas = Object.values(events);
+
+      const resolvedEvents = eventDatas.map((ed: EventData) => {
+        return ed.event;
+      }) as Events;
+
+      return resolveAttachments<Events>(resolvedEvents, attachments);
+    },
+  };
+};

inspect_ai/_view/www/src/samples/sampleLimit.ts

@@ -1,9 +1,9 @@
-import { Type11 } from "../types/log";
+import { Type14 } from "../types/log";
 
 /**
  * Formats a limit message
  */
-export const sampleLimitMessage = (type: Type11): string => {
+export const sampleLimitMessage = (type: Type14): string => {
   switch (type) {
     case "operator":
       return "Sample terminated due to operator limit.";

inspect_ai/_view/www/src/samples/scores/SampleScores.tsx

@@ -1,37 +1,22 @@
-import { FC, Fragment } from "react";
+import { FC } from "react";
 import { SampleSummary } from "../../api/types";
-import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
 
-import styles from "./SampleScores.module.css";
+import { getScoreDescriptorForValues } from "../descriptor/score/ScoreDescriptor";
 
 interface SampleScoresProps {
   sample: SampleSummary;
-  sampleDescriptor: SamplesDescriptor;
   scorer: string;
 }
 
-export const SampleScores: FC<SampleScoresProps> = ({
-  sample,
-  sampleDescriptor,
-  scorer,
-}) => {
-  const scores = scorer
-    ? sampleDescriptor.evalDescriptor
-        .scorerDescriptor(sample, { scorer, name: scorer })
-        .scores()
-    : sampleDescriptor.selectedScorerDescriptor(sample).scores();
-
-  if (scores.length === 1) {
-    return scores[0].rendered();
-  } else {
-    const rows = scores.map((score) => {
-      return (
-        <Fragment>
-          <div style={{ opacity: "0.7" }}>{score.name}</div>
-          <div>{score.rendered()}</div>
-        </Fragment>
-      );
-    });
-    return <div className={styles.grid}>{rows}</div>;
+export const SampleScores: FC<SampleScoresProps> = ({ sample, scorer }) => {
+  const scoreData = sample.scores?.[scorer];
+  if (!scoreData) {
+    return undefined;
   }
+
+  const scorerDescriptor = getScoreDescriptorForValues(
+    [scoreData.value],
+    [typeof scoreData.value],
+  );
+  return scorerDescriptor?.render(scoreData.value);
 };

inspect_ai/_view/www/src/samples/scores/SampleScoresGrid.module.css

@@ -0,0 +1,38 @@
+.container {
+  display: grid;
+  grid-template-columns:
+    minmax(auto, 1fr) minmax(auto, 1fr) minmax(auto, 1fr)
+    2fr;
+  column-gap: 0.75em;
+}
+
+.container .cell {
+  margin-bottom: 0.5em;
+}
+
+.fullWidth {
+  grid-column: 1 / -1;
+}
+
+.heading {
+  font-weight: 600;
+}
+
+.padded {
+  padding-bottom: 3em;
+}
+
+.separator {
+  height: 1px;
+  background-color: var(--bs-light-border-subtle);
+}
+
+.separatorPadded {
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+}
+
+.headerSep {
+  margin-top: 0.1em;
+  margin-bottom: 0.2em;
+}

inspect_ai/_view/www/src/samples/scores/SampleScoresGrid.tsx

@@ -0,0 +1,118 @@
+import clsx from "clsx";
+import { FC } from "react";
+import { SampleSummary } from "../../api/types";
+import { EmptyPanel } from "../../components/EmptyPanel";
+import { MetaDataGrid } from "../../metadata/MetaDataGrid";
+import { useEvalDescriptor } from "../../state/hooks";
+import { EvalSample } from "../../types/log";
+import { SampleScores } from "./SampleScores";
+import styles from "./SampleScoresGrid.module.css";
+
+interface SampleScoresGridProps {
+  evalSample: EvalSample;
+  className?: string | string[];
+}
+
+export const SampleScoresGrid: FC<SampleScoresGridProps> = ({
+  evalSample,
+  className,
+}) => {
+  const evalDescriptor = useEvalDescriptor();
+  if (!evalDescriptor) {
+    return <EmptyPanel>No Sample Selected</EmptyPanel>;
+  }
+  return (
+    <div className={clsx(className, styles.container)}>
+      <div
+        className={clsx(
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+        )}
+      >
+        Scorer
+      </div>
+      <div
+        className={clsx(
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+        )}
+      >
+        Answer
+      </div>
+      <div
+        className={clsx(
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+        )}
+      >
+        Score
+      </div>
+      <div
+        className={clsx(
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+        )}
+      >
+        Explanation
+      </div>
+      <div
+        className={clsx(styles.separator, styles.fullWidth, styles.headerSep)}
+      ></div>
+
+      {Object.keys(evalSample.scores || {}).map((scorer) => {
+        if (!evalSample.scores) {
+          return undefined;
+        }
+        const scoreData = evalSample.scores[scorer];
+        const explanation = scoreData.explanation || "(No Explanation)";
+        const answer = scoreData.answer;
+        let metadata = scoreData.metadata || {};
+
+        return (
+          <>
+            <div className={clsx("text-size-base", styles.cell)}>{scorer}</div>
+            <div className={clsx(styles.cell, "text-size-base")}>{answer}</div>
+            <div className={clsx(styles.cell, "text-size-base")}>
+              <SampleScores
+                sample={evalSample as any as SampleSummary}
+                scorer={scorer}
+              />
+            </div>
+            <div className={clsx("text-size-base", styles.cell)}>
+              {explanation}
+            </div>
+
+            {Object.keys(metadata).length > 0 ? (
+              <>
+                <div
+                  className={clsx(
+                    "text-size-smaller",
+                    "text-style-label",
+                    "text-style-secondary",
+                    styles.fullWidth,
+                  )}
+                >
+                  Metadata
+                </div>
+                <div className={clsx(styles.fullWidth)}>
+                  <MetaDataGrid entries={metadata} />
+                </div>
+                <div
+                  className={clsx(
+                    styles.separator,
+                    styles.separatorPadded,
+                    styles.fullWidth,
+                  )}
+                ></div>
+              </>
+            ) : undefined}
+          </>
+        );
+      })}
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/scores/{SampleScoreView.module.css → SampleScoresView.module.css}

@@ -1,6 +1,7 @@
 .container {
   margin-top: 0.5em;
   padding-left: 0;
+  padding-right: 0;
 }
 
 .label {
@@ -12,7 +13,7 @@
 }
 
 .wordBreak {
-  word-break: break-all;
+  word-break: break-word;
 }
 
 .scoreTable {
@@ -51,3 +52,11 @@
 .noTop {
   margin-top: 0 !important;
 }
+
+.scoreCard {
+  padding-top: 0.5em;
+}
+
+.scores {
+  padding-top: 1em;
+}

inspect_ai/_view/www/src/samples/scores/SampleScoresView.tsx

@@ -0,0 +1,78 @@
+import clsx from "clsx";
+import { Card, CardBody } from "../../components/Card";
+import { MarkdownDiv } from "../../components/MarkdownDiv";
+import { EvalSample } from "../../types/log";
+import { inputString } from "../../utils/format";
+
+import { FC } from "react";
+import ExpandablePanel from "../../components/ExpandablePanel";
+import { useEvalDescriptor } from "../../state/hooks";
+import { SampleScoresGrid } from "./SampleScoresGrid";
+import styles from "./SampleScoresView.module.css";
+
+interface SampleScoresViewProps {
+  sample?: EvalSample;
+  className?: string | string[];
+}
+
+export const SampleScoresView: FC<SampleScoresViewProps> = ({
+  sample,
+  className,
+}) => {
+  const evalDescriptor = useEvalDescriptor();
+  if (!evalDescriptor) {
+    return undefined;
+  }
+  if (!sample) {
+    return undefined;
+  }
+
+  const scoreInput = inputString(sample.input);
+  if (sample.choices && sample.choices.length > 0) {
+    scoreInput.push("");
+    scoreInput.push(
+      ...sample.choices.map((choice, index) => {
+        return `${String.fromCharCode(65 + index)}) ${choice}`;
+      }),
+    );
+  }
+
+  return (
+    <div
+      className={clsx(
+        "container-fluid",
+        className,
+        "font-size-base",
+        styles.container,
+      )}
+    >
+      <Card className={clsx(styles.scoreCard)}>
+        <CardBody>
+          <div
+            className={clsx(
+              "text-size-small",
+              "text-style-label",
+              "text-style-secondary",
+            )}
+          >
+            Input
+          </div>
+          <ExpandablePanel
+            lines={10}
+            id={`sample-score-${sample.id}-${sample.epoch}`}
+            collapse={true}
+          >
+            <MarkdownDiv
+              markdown={scoreInput.join("\n")}
+              className={clsx(styles.wordBreak, "text-size-base")}
+            />
+          </ExpandablePanel>
+          <SampleScoresGrid
+            evalSample={sample}
+            className={clsx(styles.scores)}
+          />
+        </CardBody>
+      </Card>
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/transcript/ErrorEventView.tsx

@@ -4,13 +4,10 @@ import { ANSIDisplay } from "../../components/AnsiDisplay";
 import { ErrorEvent } from "../../types/log";
 import { formatDateTime } from "../../utils/format";
 import { EventPanel } from "./event/EventPanel";
-import { TranscriptEventState } from "./types";
 
 interface ErrorEventViewProps {
   id: string;
   event: ErrorEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   className?: string | string[];
 }
 
@@ -20,8 +17,6 @@ interface ErrorEventViewProps {
 export const ErrorEventView: FC<ErrorEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   className,
 }) => {
   return (
@@ -31,14 +26,6 @@ export const ErrorEventView: FC<ErrorEventViewProps> = ({
       className={className}
       subTitle={formatDateTime(new Date(event.timestamp))}
       icon={ApplicationIcons.error}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav: string) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed: boolean) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       <ANSIDisplay
         output={event.error.traceback_ansi}

inspect_ai/_view/www/src/samples/transcript/InfoEventView.tsx

@@ -6,13 +6,10 @@ import { InfoEvent } from "../../types/log";
 import { formatDateTime } from "../../utils/format";
 import { EventPanel } from "./event/EventPanel";
 import styles from "./InfoEventView.module.css";
-import { TranscriptEventState } from "./types";
 
 interface InfoEventViewProps {
   id: string;
   event: InfoEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   className?: string | string[];
 }
 
@@ -22,8 +19,6 @@ interface InfoEventViewProps {
 export const InfoEventView: FC<InfoEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   className,
 }) => {
   const panels = [];
@@ -40,14 +35,6 @@ export const InfoEventView: FC<InfoEventViewProps> = ({
       className={className}
       subTitle={formatDateTime(new Date(event.timestamp))}
       icon={ApplicationIcons.info}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       {panels}
     </EventPanel>

inspect_ai/_view/www/src/samples/transcript/InputEventView.tsx

@@ -4,13 +4,10 @@ import { ANSIDisplay } from "../../components/AnsiDisplay";
 import { InputEvent } from "../../types/log";
 import { formatDateTime } from "../../utils/format";
 import { EventPanel } from "./event/EventPanel";
-import { TranscriptEventState } from "./types";
 
 interface InputEventViewProps {
   id: string;
   event: InputEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   className?: string | string[];
 }
 
@@ -20,8 +17,6 @@ interface InputEventViewProps {
 export const InputEventView: FC<InputEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   className,
 }) => {
   return (
@@ -31,14 +26,6 @@ export const InputEventView: FC<InputEventViewProps> = ({
       className={className}
       subTitle={formatDateTime(new Date(event.timestamp))}
       icon={ApplicationIcons.input}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       <ANSIDisplay
         output={event.input_ansi}

inspect_ai/_view/www/src/samples/transcript/ModelEventView.module.css

@@ -41,3 +41,7 @@
   column-gap: 1em;
   row-gap: 0.5em;
 }
+
+.progress {
+  margin-left: 2em;
+}

inspect_ai/_view/www/src/samples/transcript/ModelEventView.tsx

@@ -3,7 +3,7 @@ import "prismjs/components/prism-json";
 import "prismjs/components/prism-python";
 
 import clsx from "clsx";
-import { FC, Fragment, useEffect, useMemo, useRef } from "react";
+import { FC, Fragment, useMemo } from "react";
 import { ApplicationIcons } from "../../appearance/icons";
 import { MetaDataGrid } from "../../metadata/MetaDataGrid";
 import {
@@ -17,9 +17,9 @@ import { ModelUsagePanel } from "../../usage/ModelUsagePanel";
 import { ChatView } from "../chat/ChatView";
 import { EventPanel } from "./event/EventPanel";
 import { EventSection } from "./event/EventSection";
-import { TranscriptEventState } from "./types";
 
-import { highlightElement } from "prismjs";
+import { PulsingDots } from "../../components/PulsingDots";
+import { usePrismHighlight } from "../../state/hooks";
 import styles from "./ModelEventView.module.css";
 import { EventTimingPanel } from "./event/EventTimingPanel";
 import { formatTiming, formatTitle } from "./event/utils";
@@ -27,8 +27,6 @@ import { formatTiming, formatTitle } from "./event/utils";
 interface ModelEventViewProps {
   id: string;
   event: ModelEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   className?: string | string[];
 }
 
@@ -38,8 +36,6 @@ interface ModelEventViewProps {
 export const ModelEventView: FC<ModelEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   className,
 }) => {
   const totalUsage = event.output.usage?.total_tokens;
@@ -74,14 +70,6 @@ export const ModelEventView: FC<ModelEventViewProps> = ({
       title={formatTitle(`Model Call: ${event.model}`, totalUsage, callTime)}
       subTitle={formatTiming(event.timestamp, event.working_start)}
       icon={ApplicationIcons.model}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       <div data-name="Summary" className={styles.container}>
         <ChatView
@@ -91,6 +79,11 @@ export const ModelEventView: FC<ModelEventViewProps> = ({
           numbered={false}
           toolCallStyle="compact"
         />
+        {event.pending ? (
+          <div className={clsx(styles.progress)}>
+            <PulsingDots subtle={false} size="medium" />
+          </div>
+        ) : undefined}
       </div>
       <div data-name="All" className={styles.container}>
         <div className={styles.all}>
@@ -170,27 +163,20 @@ interface APICodeCellProps {
 }
 
 export const APICodeCell: FC<APICodeCellProps> = ({ id, contents }) => {
-  const codeRef = useRef<HTMLElement>(null);
   const sourceCode = useMemo(() => {
     return JSON.stringify(contents, undefined, 2);
   }, [contents]);
-
-  useEffect(() => {
-    if (codeRef.current) {
-      highlightElement(codeRef.current);
-    }
-  }, [contents]);
+  const prismParentRef = usePrismHighlight(sourceCode);
 
   if (!contents) {
     return null;
   }
 
   return (
-    <div className={clsx("model-call")}>
+    <div ref={prismParentRef} className={clsx("model-call")}>
       <pre className={clsx(styles.codePre)}>
         <code
           id={id}
-          ref={codeRef}
           className={clsx("language-json", styles.code, "text-size-small")}
         >
           {sourceCode}