inspect-ai 0.3.57__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs

@@ -1,5 +1,6 @@
 // @ts-check
 import { html } from "htm/preact";
+import { useCallback, useState } from "preact/hooks";
 import { SampleInitEventView } from "./SampleInitEventView.mjs";
 import { StateEventView } from "./state/StateEventView.mjs";
 import { StepEventView } from "./StepEventView.mjs";
@@ -15,6 +16,8 @@ import { ApprovalEventView } from "./ApprovalEventView.mjs";
 import { SampleLimitEventView } from "./SampleLimitEventView.mjs";
 import { FontSize } from "../../appearance/Fonts.mjs";
 import { EventNode } from "./Types.mjs";
+// @ts-ignore
+import { VirtualList } from "../../components/VirtualList.mjs";
 
 /**
  * Renders the TranscriptView component.
@@ -26,10 +29,61 @@ import { EventNode } from "./Types.mjs";
  * @returns {import("preact").JSX.Element} The TranscriptView component.
  */
 export const TranscriptView = ({ id, events, depth = 0 }) => {
+  const [transcriptState, setTranscriptState] = useState({});
+  const onTranscriptState = useCallback(
+    (state) => {
+      setTranscriptState(state);
+    },
+    [transcriptState, setTranscriptState],
+  );
+
   // Normalize Events themselves
   const resolvedEvents = fixupEventStream(events);
   const eventNodes = treeifyEvents(resolvedEvents, depth);
-  return html` <${TranscriptComponent} id=${id} eventNodes=${eventNodes} /> `;
+  return html`
+    <${TranscriptComponent}
+      id=${id}
+      eventNodes=${eventNodes}
+      transcriptState=${transcriptState}
+      setTranscriptState=${onTranscriptState}
+    />
+  `;
+};
+
+/**
+ * Renders the Transcript component.
+ *
+ * @param {Object} props - The parameters for the component.
+ * @param {string} props.id - The identifier for this view
+ * @param {import("../../types/log").Events} props.events - The transcript events to display.
+ * @param {Object} props.style - The transcript style to display.
+ * @param {number} props.depth - The base depth for this transcript view
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable parent element
+ * @returns {import("preact").JSX.Element} The TranscriptView component.
+ */
+export const TranscriptVirtualList = (props) => {
+  let { id, scrollRef, events, depth, style } = props;
+
+  // Normalize Events themselves
+  const resolvedEvents = fixupEventStream(events);
+  const eventNodes = treeifyEvents(resolvedEvents, depth);
+
+  const [transcriptState, setTranscriptState] = useState({});
+  const onTranscriptState = useCallback(
+    (state) => {
+      setTranscriptState(state);
+    },
+    [transcriptState, setTranscriptState],
+  );
+
+  return html`<${TranscriptVirtualListComponent}
+    id=${id}
+    eventNodes=${eventNodes}
+    style=${style}
+    scrollRef=${scrollRef}
+    transcriptState=${transcriptState}
+    setTranscriptState=${onTranscriptState}
+  />`;
 };
 
 /**
@@ -39,9 +93,81 @@ export const TranscriptView = ({ id, events, depth = 0 }) => {
  * @param {string} props.id - The identifier for this view
  * @param {EventNode[]} props.eventNodes - The transcript events nodes to display.
  * @param {Object} props.style - The transcript style to display.
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable parent element
+ * @param {import("./Types.mjs").TranscriptState} props.transcriptState - The state for this transcript
+ * @param {(state: import("./Types.mjs").TranscriptState) => void} props.setTranscriptState - Set the transcript state for this transcript
  * @returns {import("preact").JSX.Element} The TranscriptView component.
  */
-export const TranscriptComponent = ({ id, eventNodes, style }) => {
+export const TranscriptVirtualListComponent = ({
+  id,
+  eventNodes,
+  style,
+  scrollRef,
+  transcriptState,
+  setTranscriptState,
+}) => {
+  const renderRow = (item, index) => {
+    const toggleStyle = {};
+    if (item.depth % 2 == 0) {
+      toggleStyle.backgroundColor = "var(--bs-light-bg-subtle)";
+    } else {
+      toggleStyle.backgroundColor = "var(--bs-body-bg)";
+    }
+
+    let paddingTop = "0";
+    if (index === 0) {
+      paddingTop = ".5em";
+    }
+    const eventId = `${id}-event${index}`;
+    const setEventState = useCallback(
+      (state) => {
+        setTranscriptState({ ...transcriptState, [eventId]: state });
+      },
+      [setTranscriptState, transcriptState],
+    );
+
+    return html`<div style=${{ paddingTop, paddingBottom: ".5em" }}>
+      <${RenderedEventNode}
+        id=${eventId}
+        node=${item}
+        style=${{
+          ...toggleStyle,
+          ...style,
+        }}
+        scrollRef=${scrollRef}
+        eventState=${transcriptState[eventId] || {}}
+        setEventState=${setEventState}
+      />
+    </div>`;
+  };
+
+  return html`<${VirtualList}
+    data=${eventNodes}
+    tabIndex="0"
+    renderRow=${renderRow}
+    scrollRef=${scrollRef}
+    style=${{ width: "100%", marginTop: "1em" }}
+  />`;
+};
+
+/**
+ * Renders the Transcript component.
+ *
+ * @param {Object} props - The parameters for the component.
+ * @param {string} props.id - The identifier for this view
+ * @param {EventNode[]} props.eventNodes - The transcript events nodes to display.
+ * @param {Object} props.style - The transcript style to display.
+ * @param {import("./Types.mjs").TranscriptState} props.transcriptState - The state for this transcript
+ * @param {(state: import("./Types.mjs").TranscriptState) => void} props.setTranscriptState - Set the transcript state for this transcript
+ * @returns {import("preact").JSX.Element} The TranscriptView component.
+ */
+export const TranscriptComponent = ({
+  id,
+  transcriptState,
+  setTranscriptState,
+  eventNodes,
+  style,
+}) => {
   const rows = eventNodes.map((eventNode, index) => {
     const toggleStyle = {};
     if (eventNode.depth % 2 == 0) {
@@ -55,15 +181,32 @@ export const TranscriptComponent = ({ id, eventNodes, style }) => {
       toggleStyle.marginBottom = "1.5em";
     }
 
+    let paddingBottom = ".5em";
+    if (index === eventNodes.length - 1) {
+      paddingBottom = "0";
+    }
+
+    const eventId = `${id}-event${index}`;
+    const setEventState = useCallback(
+      (state) => {
+        setTranscriptState({ ...transcriptState, [eventId]: state });
+      },
+      [setTranscriptState, transcriptState],
+    );
+
     const row = html`
-      <${RenderedEventNode}
-        id=${`${id}-event${index}`}
-        node=${eventNode}
-        style=${{
-          ...toggleStyle,
-          ...style,
-        }}
-      />
+      <div style=${{ paddingBottom }}>
+        <${RenderedEventNode}
+          id=${eventId}
+          node=${eventNode}
+          style=${{
+            ...toggleStyle,
+            ...style,
+          }}
+          eventState=${transcriptState[eventId] || {}}
+          setEventState=${setEventState}
+        />
+      </div>
     `;
     return row;
   });
@@ -89,14 +232,26 @@ export const TranscriptComponent = ({ id, eventNodes, style }) => {
  * @param {string} props.id - The id for this event.
  * @param { EventNode } props.node - This event.
  * @param { Object } props.style - The style for this node.
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable parent element
+ * @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
+ * @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
  * @returns {import("preact").JSX.Element} The rendered event.
  */
-export const RenderedEventNode = ({ id, node, style }) => {
+export const RenderedEventNode = ({
+  id,
+  node,
+  style,
+  scrollRef,
+  eventState,
+  setEventState,
+}) => {
   switch (node.event.event) {
     case "sample_init":
       return html`<${SampleInitEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -104,6 +259,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${SampleLimitEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -111,6 +268,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${InfoEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -118,6 +277,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${LoggerEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -125,6 +286,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ModelEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -132,6 +295,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ScoreEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -139,6 +304,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${StateEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -146,14 +313,19 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${StepEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         children=${node.children}
         style=${style}
+        scrollRef=${scrollRef}
       />`;
 
     case "store":
       return html`<${StateEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
         isStore=${true}
       />`;
@@ -162,6 +334,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${SubtaskEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
         depth=${node.depth}
       />`;
@@ -170,6 +344,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ToolEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
         depth=${node.depth}
       />`;
@@ -178,6 +354,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${InputEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -185,6 +363,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ErrorEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -192,6 +372,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ApprovalEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 

inspect_ai/_view/www/src/samples/transcript/Types.mjs

@@ -32,3 +32,13 @@ export class EventNode {
     this.depth = depth;
   }
 }
+
+/**
+ * @typedef {Record<string, TranscriptEventState>} TranscriptState
+ */
+
+/**
+ * @typedef {Object} TranscriptEventState
+ * @property {string} [selectedNav] - The selected nav for this event
+ * @property {boolean} [collapsed] - The collapse state for this event
+ */

inspect_ai/_view/www/src/samples/transcript/state/StateEventView.mjs

@@ -15,11 +15,20 @@ import { formatDateTime } from "../../../utils/Format.mjs";
  * @param {Object} props - The properties passed to the component.
  * @param { string  } props.id - The id of this event.
  * @param {import("../../../types/log").StateEvent } props.event - The event object to display.
+ * @param {import("./../Types.mjs").TranscriptEventState} props.eventState - The state for this event
+ * @param {(state: import("./../Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
  * @param { boolean } props.isStore - Whether this event view is rendering a storage (rather than a state)
  * @param { Object } props.style - The style of this event.
  * @returns {import("preact").JSX.Element} The component.
  */
-export const StateEventView = ({ id, event, isStore, style }) => {
+export const StateEventView = ({
+  id,
+  event,
+  eventState,
+  setEventState,
+  isStore,
+  style,
+}) => {
   const summary = summarizeChanges(event.changes);
 
   // Synthesize objects for comparison
@@ -53,7 +62,22 @@ export const StateEventView = ({ id, event, isStore, style }) => {
   const title = event.event === "state" ? "State Updated" : "Store Updated";
 
   return html`
-  <${EventPanel} id=${id} title="${title}" subTitle=${formatDateTime(new Date(event.timestamp))} text=${tabs.length === 1 ? summary : undefined} collapse=${changePreview === undefined ? true : undefined} style=${style}>
+  <${EventPanel} 
+    id=${id} 
+    title="${title}" 
+    subTitle=${formatDateTime(new Date(event.timestamp))} 
+    text=${tabs.length === 1 ? summary : undefined} 
+    collapse=${changePreview === undefined ? true : undefined} 
+    style=${style}
+    selectedNav=${eventState.selectedNav || ""}
+    onSelectedNav=${(selectedNav) => {
+      setEventState({ ...eventState, selectedNav });
+    }}
+    collapsed=${eventState.collapsed}
+    onCollapsed=${(collapsed) => {
+      setEventState({ ...eventState, collapsed });
+    }}                  
+  >
     ${tabs}
   </${EventPanel}>`;
 };

inspect_ai/_view/www/src/types/log.d.ts

@@ -32,7 +32,6 @@ export type Limit = number | [unknown, unknown] | null;
 export type SampleId = string | number | (string | number)[] | null;
 export type Epochs = number | null;
 export type EpochsReducer = string[] | null;
-export type Trace = boolean | null;
 export type Name1 = string;
 export type Tools = string | string[];
 export type Approvers = ApproverPolicyConfig[];
@@ -77,6 +76,7 @@ export type NumChoices = number | null;
 export type Logprobs = boolean | null;
 export type TopLogprobs = number | null;
 export type ParallelToolCalls = boolean | null;
+export type InternalTools = boolean | null;
 export type MaxToolOutput = number | null;
 export type CachePrompt = "auto" | boolean | null;
 export type ReasoningEffort = ("low" | "medium" | "high") | null;
@@ -112,35 +112,49 @@ export type Input =
       | ChatMessageAssistant
       | ChatMessageTool
     )[];
-export type Content = string | (ContentText | ContentImage)[];
+export type Content =
+  | string
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Type1 = "text";
 export type Text = string;
 export type Type2 = "image";
 export type Image = string;
 export type Detail = "auto" | "low" | "high";
+export type Type3 = "audio";
+export type Audio = string;
+export type Format = "wav" | "mp3";
+export type Type4 = "video";
+export type Video = string;
+export type Format1 = "mp4" | "mpeg" | "mov";
 export type Source = ("input" | "generate") | null;
 export type Role = "system";
-export type Content1 = string | (ContentText | ContentImage)[];
+export type Content1 =
+  | string
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source1 = ("input" | "generate") | null;
 export type Role1 = "user";
 export type ToolCallId = string | null;
-export type Content2 = string | (ContentText | ContentImage)[];
+export type Content2 =
+  | string
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source2 = ("input" | "generate") | null;
 export type Role2 = "assistant";
 export type ToolCalls = ToolCall[] | null;
 export type Id1 = string;
 export type Function = string;
-export type Type3 = "function";
+export type Type5 = "function";
 export type ParseError = string | null;
 export type Title = string | null;
-export type Format = "text" | "markdown";
+export type Format2 = "text" | "markdown";
 export type Content3 = string;
-export type Content4 = string | (ContentText | ContentImage)[];
+export type Content4 =
+  | string
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source3 = ("input" | "generate") | null;
 export type Role3 = "tool";
 export type ToolCallId1 = string | null;
 export type Function1 = string | null;
-export type Type4 =
+export type Type6 =
   | "parsing"
   | "timeout"
   | "unicode_decode"
@@ -218,7 +232,7 @@ export type JsonValue = unknown;
 export type Timestamp1 = string;
 export type Pending1 = boolean | null;
 export type Event1 = "sample_limit";
-export type Type5 = "message" | "time" | "token" | "operator";
+export type Type7 = "message" | "time" | "token" | "operator" | "custom";
 export type Message2 = string;
 export type Limit1 = number | null;
 export type Timestamp2 = string;
@@ -244,8 +258,8 @@ export type Input2 = (
 )[];
 export type Name5 = string;
 export type Description = string;
-export type Type6 = "object";
-export type Type7 =
+export type Type8 = "object";
+export type Type9 =
   | ("string" | "integer" | "number" | "boolean" | "array" | "object" | "null")
   | null;
 export type Description1 = string | null;
@@ -261,11 +275,12 @@ export type Additionalproperties1 = boolean;
 export type Tools1 = ToolInfo[];
 export type ToolChoice = ("auto" | "any" | "none") | ToolFunction;
 export type Name6 = string;
+export type Error1 = string | null;
 export type Cache = ("read" | "write") | null;
 export type Timestamp5 = string;
 export type Pending5 = boolean | null;
 export type Event5 = "tool";
-export type Type8 = "function";
+export type Type10 = "function";
 export type Id3 = string;
 export type Function2 = string;
 export type Result =
@@ -274,7 +289,9 @@ export type Result =
   | boolean
   | ContentText
   | ContentImage
-  | (ContentText | ContentImage)[];
+  | ContentAudio
+  | ContentVideo
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Truncated = [unknown, unknown] | null;
 export type Timestamp6 = string;
 export type Pending6 = boolean | null;
@@ -324,13 +341,13 @@ export type Timestamp12 = string;
 export type Pending12 = boolean | null;
 export type Event12 = "step";
 export type Action = "begin" | "end";
-export type Type9 = string | null;
+export type Type11 = string | null;
 export type Name8 = string;
 export type Timestamp13 = string;
 export type Pending13 = boolean | null;
 export type Event13 = "subtask";
 export type Name9 = string;
-export type Type10 = string | null;
+export type Type12 = string | null;
 export type Events2 = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -379,7 +396,13 @@ export type Events = (
   | StepEvent
   | SubtaskEvent
 )[];
-export type Type11 = "context" | "time" | "message" | "token" | "operator";
+export type Type13 =
+  | "context"
+  | "time"
+  | "message"
+  | "token"
+  | "operator"
+  | "custom";
 export type Limit2 = number;
 export type Reductions = EvalSampleReductions[] | null;
 export type Scorer1 = string;
@@ -448,7 +471,6 @@ export interface EvalConfig {
   sample_id: SampleId;
   epochs: Epochs;
   epochs_reducer: EpochsReducer;
-  trace: Trace;
   approval: ApprovalPolicyConfig | null;
   fail_on_error: FailOnError;
   message_limit: MessageLimit;
@@ -531,6 +553,7 @@ export interface GenerateConfig {
   logprobs: Logprobs;
   top_logprobs: TopLogprobs;
   parallel_tool_calls: ParallelToolCalls;
+  internal_tools: InternalTools;
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
@@ -614,6 +637,16 @@ export interface ContentImage {
   image: Image;
   detail: Detail;
 }
+export interface ContentAudio {
+  type: Type3;
+  audio: Audio;
+  format: Format;
+}
+export interface ContentVideo {
+  type: Type4;
+  video: Video;
+  format: Format1;
+}
 export interface ChatMessageUser {
   content: Content1;
   source: Source1;
@@ -630,7 +663,7 @@ export interface ToolCall {
   id: Id1;
   function: Function;
   arguments: Arguments;
-  type: Type3;
+  type: Type5;
   parse_error: ParseError;
   view: ToolCallContent | null;
 }
@@ -640,7 +673,7 @@ export interface Arguments {}
  */
 export interface ToolCallContent {
   title: Title;
-  format: Format;
+  format: Format2;
   content: Content3;
 }
 export interface ChatMessageTool {
@@ -652,7 +685,7 @@ export interface ChatMessageTool {
   error: ToolCallError | null;
 }
 export interface ToolCallError {
-  type: Type4;
+  type: Type6;
   message: Message1;
 }
 export interface ModelOutput {
@@ -735,7 +768,7 @@ export interface SampleLimitEvent {
   timestamp: Timestamp1;
   pending: Pending1;
   event: Event1;
-  type: Type5;
+  type: Type7;
   message: Message2;
   limit: Limit1;
 }
@@ -784,6 +817,7 @@ export interface ModelEvent {
   tool_choice: ToolChoice;
   config: GenerateConfig1;
   output: ModelOutput;
+  error: Error1;
   cache: Cache;
   call: ModelCall | null;
 }
@@ -822,7 +856,7 @@ export interface ToolInfo {
  * Description of tool parameters object in JSON Schema format.
  */
 export interface ToolParams {
-  type: Type6;
+  type: Type8;
   properties: Properties;
   required: Required1;
   additionalProperties: Additionalproperties1;
@@ -834,7 +868,7 @@ export interface Properties {
  * Description of tool parameter in JSON Schema format.
  */
 export interface ToolParam {
-  type: Type7;
+  type: Type9;
   description: Description1;
   default: Default;
   enum: Enum;
@@ -873,6 +907,7 @@ export interface GenerateConfig1 {
   logprobs: Logprobs;
   top_logprobs: TopLogprobs;
   parallel_tool_calls: ParallelToolCalls;
+  internal_tools: InternalTools;
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
@@ -897,7 +932,7 @@ export interface ToolEvent {
   timestamp: Timestamp5;
   pending: Pending5;
   event: Event5;
-  type: Type8;
+  type: Type10;
   id: Id3;
   function: Function2;
   arguments: Arguments1;
@@ -999,7 +1034,7 @@ export interface StepEvent {
   pending: Pending12;
   event: Event12;
   action: Action;
-  type: Type9;
+  type: Type11;
   name: Name8;
 }
 /**
@@ -1010,7 +1045,7 @@ export interface SubtaskEvent {
   pending: Pending13;
   event: Event13;
   name: Name9;
-  type: Type10;
+  type: Type12;
   input: Input4;
   result: Result1;
   events: Events2;
@@ -1026,7 +1061,7 @@ export interface Attachments {
   [k: string]: string;
 }
 export interface EvalSampleLimit {
-  type: Type11;
+  type: Type13;
   limit: Limit2;
 }
 export interface EvalSampleReductions {