inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl

inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs

@@ -1,5 +1,6 @@
 // @ts-check
 import { html } from "htm/preact";
+import { useCallback, useState } from "preact/hooks";
 import { SampleInitEventView } from "./SampleInitEventView.mjs";
 import { StateEventView } from "./state/StateEventView.mjs";
 import { StepEventView } from "./StepEventView.mjs";
@@ -15,6 +16,8 @@ import { ApprovalEventView } from "./ApprovalEventView.mjs";
 import { SampleLimitEventView } from "./SampleLimitEventView.mjs";
 import { FontSize } from "../../appearance/Fonts.mjs";
 import { EventNode } from "./Types.mjs";
+// @ts-ignore
+import { VirtualList } from "../../components/VirtualList.mjs";
 
 /**
  * Renders the TranscriptView component.
@@ -26,10 +29,61 @@ import { EventNode } from "./Types.mjs";
  * @returns {import("preact").JSX.Element} The TranscriptView component.
  */
 export const TranscriptView = ({ id, events, depth = 0 }) => {
+  const [transcriptState, setTranscriptState] = useState({});
+  const onTranscriptState = useCallback(
+    (state) => {
+      setTranscriptState(state);
+    },
+    [transcriptState, setTranscriptState],
+  );
+
   // Normalize Events themselves
   const resolvedEvents = fixupEventStream(events);
   const eventNodes = treeifyEvents(resolvedEvents, depth);
-  return html` <${TranscriptComponent} id=${id} eventNodes=${eventNodes} /> `;
+  return html`
+    <${TranscriptComponent}
+      id=${id}
+      eventNodes=${eventNodes}
+      transcriptState=${transcriptState}
+      setTranscriptState=${onTranscriptState}
+    />
+  `;
+};
+
+/**
+ * Renders the Transcript component.
+ *
+ * @param {Object} props - The parameters for the component.
+ * @param {string} props.id - The identifier for this view
+ * @param {import("../../types/log").Events} props.events - The transcript events to display.
+ * @param {Object} props.style - The transcript style to display.
+ * @param {number} props.depth - The base depth for this transcript view
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable parent element
+ * @returns {import("preact").JSX.Element} The TranscriptView component.
+ */
+export const TranscriptVirtualList = (props) => {
+  let { id, scrollRef, events, depth, style } = props;
+
+  // Normalize Events themselves
+  const resolvedEvents = fixupEventStream(events);
+  const eventNodes = treeifyEvents(resolvedEvents, depth);
+
+  const [transcriptState, setTranscriptState] = useState({});
+  const onTranscriptState = useCallback(
+    (state) => {
+      setTranscriptState(state);
+    },
+    [transcriptState, setTranscriptState],
+  );
+
+  return html`<${TranscriptVirtualListComponent}
+    id=${id}
+    eventNodes=${eventNodes}
+    style=${style}
+    scrollRef=${scrollRef}
+    transcriptState=${transcriptState}
+    setTranscriptState=${onTranscriptState}
+  />`;
 };
 
 /**
@@ -39,9 +93,81 @@ export const TranscriptView = ({ id, events, depth = 0 }) => {
  * @param {string} props.id - The identifier for this view
  * @param {EventNode[]} props.eventNodes - The transcript events nodes to display.
  * @param {Object} props.style - The transcript style to display.
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable parent element
+ * @param {import("./Types.mjs").TranscriptState} props.transcriptState - The state for this transcript
+ * @param {(state: import("./Types.mjs").TranscriptState) => void} props.setTranscriptState - Set the transcript state for this transcript
  * @returns {import("preact").JSX.Element} The TranscriptView component.
  */
-export const TranscriptComponent = ({ id, eventNodes, style }) => {
+export const TranscriptVirtualListComponent = ({
+  id,
+  eventNodes,
+  style,
+  scrollRef,
+  transcriptState,
+  setTranscriptState,
+}) => {
+  const renderRow = (item, index) => {
+    const toggleStyle = {};
+    if (item.depth % 2 == 0) {
+      toggleStyle.backgroundColor = "var(--bs-light-bg-subtle)";
+    } else {
+      toggleStyle.backgroundColor = "var(--bs-body-bg)";
+    }
+
+    let paddingTop = "0";
+    if (index === 0) {
+      paddingTop = ".5em";
+    }
+    const eventId = `${id}-event${index}`;
+    const setEventState = useCallback(
+      (state) => {
+        setTranscriptState({ ...transcriptState, [eventId]: state });
+      },
+      [setTranscriptState, transcriptState],
+    );
+
+    return html`<div style=${{ paddingTop, paddingBottom: ".5em" }}>
+      <${RenderedEventNode}
+        id=${eventId}
+        node=${item}
+        style=${{
+          ...toggleStyle,
+          ...style,
+        }}
+        scrollRef=${scrollRef}
+        eventState=${transcriptState[eventId] || {}}
+        setEventState=${setEventState}
+      />
+    </div>`;
+  };
+
+  return html`<${VirtualList}
+    data=${eventNodes}
+    tabIndex="0"
+    renderRow=${renderRow}
+    scrollRef=${scrollRef}
+    style=${{ width: "100%", marginTop: "1em" }}
+  />`;
+};
+
+/**
+ * Renders the Transcript component.
+ *
+ * @param {Object} props - The parameters for the component.
+ * @param {string} props.id - The identifier for this view
+ * @param {EventNode[]} props.eventNodes - The transcript events nodes to display.
+ * @param {Object} props.style - The transcript style to display.
+ * @param {import("./Types.mjs").TranscriptState} props.transcriptState - The state for this transcript
+ * @param {(state: import("./Types.mjs").TranscriptState) => void} props.setTranscriptState - Set the transcript state for this transcript
+ * @returns {import("preact").JSX.Element} The TranscriptView component.
+ */
+export const TranscriptComponent = ({
+  id,
+  transcriptState,
+  setTranscriptState,
+  eventNodes,
+  style,
+}) => {
   const rows = eventNodes.map((eventNode, index) => {
     const toggleStyle = {};
     if (eventNode.depth % 2 == 0) {
@@ -55,15 +181,32 @@ export const TranscriptComponent = ({ id, eventNodes, style }) => {
       toggleStyle.marginBottom = "1.5em";
     }
 
+    let paddingBottom = ".5em";
+    if (index === eventNodes.length - 1) {
+      paddingBottom = "0";
+    }
+
+    const eventId = `${id}-event${index}`;
+    const setEventState = useCallback(
+      (state) => {
+        setTranscriptState({ ...transcriptState, [eventId]: state });
+      },
+      [setTranscriptState, transcriptState],
+    );
+
     const row = html`
-      <${RenderedEventNode}
-        id=${`${id}-event${index}`}
-        node=${eventNode}
-        style=${{
-          ...toggleStyle,
-          ...style,
-        }}
-      />
+      <div style=${{ paddingBottom }}>
+        <${RenderedEventNode}
+          id=${eventId}
+          node=${eventNode}
+          style=${{
+            ...toggleStyle,
+            ...style,
+          }}
+          eventState=${transcriptState[eventId] || {}}
+          setEventState=${setEventState}
+        />
+      </div>
     `;
     return row;
   });
@@ -89,14 +232,26 @@ export const TranscriptComponent = ({ id, eventNodes, style }) => {
  * @param {string} props.id - The id for this event.
  * @param { EventNode } props.node - This event.
  * @param { Object } props.style - The style for this node.
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable parent element
+ * @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
+ * @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
  * @returns {import("preact").JSX.Element} The rendered event.
  */
-export const RenderedEventNode = ({ id, node, style }) => {
+export const RenderedEventNode = ({
+  id,
+  node,
+  style,
+  scrollRef,
+  eventState,
+  setEventState,
+}) => {
   switch (node.event.event) {
     case "sample_init":
       return html`<${SampleInitEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -104,6 +259,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${SampleLimitEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -111,6 +268,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${InfoEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -118,6 +277,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${LoggerEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -125,6 +286,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ModelEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -132,6 +295,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ScoreEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -139,6 +304,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${StateEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -146,14 +313,19 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${StepEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         children=${node.children}
         style=${style}
+        scrollRef=${scrollRef}
       />`;
 
     case "store":
       return html`<${StateEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
         isStore=${true}
       />`;
@@ -162,6 +334,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${SubtaskEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
         depth=${node.depth}
       />`;
@@ -170,6 +344,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ToolEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
         depth=${node.depth}
       />`;
@@ -178,6 +354,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${InputEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -185,6 +363,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ErrorEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 
@@ -192,6 +372,8 @@ export const RenderedEventNode = ({ id, node, style }) => {
       return html`<${ApprovalEventView}
         id=${id}
         event=${node.event}
+        eventState=${eventState}
+        setEventState=${setEventState}
         style=${style}
       />`;
 

inspect_ai/_view/www/src/samples/transcript/Types.mjs

@@ -32,3 +32,13 @@ export class EventNode {
     this.depth = depth;
   }
 }
+
+/**
+ * @typedef {Record<string, TranscriptEventState>} TranscriptState
+ */
+
+/**
+ * @typedef {Object} TranscriptEventState
+ * @property {string} [selectedNav] - The selected nav for this event
+ * @property {boolean} [collapsed] - The collapse state for this event
+ */

inspect_ai/_view/www/src/samples/transcript/state/StateEventView.mjs

@@ -15,11 +15,20 @@ import { formatDateTime } from "../../../utils/Format.mjs";
  * @param {Object} props - The properties passed to the component.
  * @param { string  } props.id - The id of this event.
  * @param {import("../../../types/log").StateEvent } props.event - The event object to display.
+ * @param {import("./../Types.mjs").TranscriptEventState} props.eventState - The state for this event
+ * @param {(state: import("./../Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
  * @param { boolean } props.isStore - Whether this event view is rendering a storage (rather than a state)
  * @param { Object } props.style - The style of this event.
  * @returns {import("preact").JSX.Element} The component.
  */
-export const StateEventView = ({ id, event, isStore, style }) => {
+export const StateEventView = ({
+  id,
+  event,
+  eventState,
+  setEventState,
+  isStore,
+  style,
+}) => {
   const summary = summarizeChanges(event.changes);
 
   // Synthesize objects for comparison
@@ -53,7 +62,22 @@ export const StateEventView = ({ id, event, isStore, style }) => {
   const title = event.event === "state" ? "State Updated" : "Store Updated";
 
   return html`
-  <${EventPanel} id=${id} title="${title}" subTitle=${formatDateTime(new Date(event.timestamp))} text=${tabs.length === 1 ? summary : undefined} collapse=${changePreview === undefined ? true : undefined} style=${style}>
+  <${EventPanel} 
+    id=${id} 
+    title="${title}" 
+    subTitle=${formatDateTime(new Date(event.timestamp))} 
+    text=${tabs.length === 1 ? summary : undefined} 
+    collapse=${changePreview === undefined ? true : undefined} 
+    style=${style}
+    selectedNav=${eventState.selectedNav || ""}
+    onSelectedNav=${(selectedNav) => {
+      setEventState({ ...eventState, selectedNav });
+    }}
+    collapsed=${eventState.collapsed}
+    onCollapsed=${(collapsed) => {
+      setEventState({ ...eventState, collapsed });
+    }}                  
+  >
     ${tabs}
   </${EventPanel}>`;
 };

inspect_ai/_view/www/src/types/log.d.ts

@@ -76,6 +76,7 @@ export type NumChoices = number | null;
 export type Logprobs = boolean | null;
 export type TopLogprobs = number | null;
 export type ParallelToolCalls = boolean | null;
+export type InternalTools = boolean | null;
 export type MaxToolOutput = number | null;
 export type CachePrompt = "auto" | boolean | null;
 export type ReasoningEffort = ("low" | "medium" | "high") | null;
@@ -231,7 +232,7 @@ export type JsonValue = unknown;
 export type Timestamp1 = string;
 export type Pending1 = boolean | null;
 export type Event1 = "sample_limit";
-export type Type7 = "message" | "time" | "token" | "operator";
+export type Type7 = "message" | "time" | "token" | "operator" | "custom";
 export type Message2 = string;
 export type Limit1 = number | null;
 export type Timestamp2 = string;
@@ -274,6 +275,7 @@ export type Additionalproperties1 = boolean;
 export type Tools1 = ToolInfo[];
 export type ToolChoice = ("auto" | "any" | "none") | ToolFunction;
 export type Name6 = string;
+export type Error1 = string | null;
 export type Cache = ("read" | "write") | null;
 export type Timestamp5 = string;
 export type Pending5 = boolean | null;
@@ -394,7 +396,13 @@ export type Events = (
   | StepEvent
   | SubtaskEvent
 )[];
-export type Type13 = "context" | "time" | "message" | "token" | "operator";
+export type Type13 =
+  | "context"
+  | "time"
+  | "message"
+  | "token"
+  | "operator"
+  | "custom";
 export type Limit2 = number;
 export type Reductions = EvalSampleReductions[] | null;
 export type Scorer1 = string;
@@ -545,6 +553,7 @@ export interface GenerateConfig {
   logprobs: Logprobs;
   top_logprobs: TopLogprobs;
   parallel_tool_calls: ParallelToolCalls;
+  internal_tools: InternalTools;
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
@@ -808,6 +817,7 @@ export interface ModelEvent {
   tool_choice: ToolChoice;
   config: GenerateConfig1;
   output: ModelOutput;
+  error: Error1;
   cache: Cache;
   call: ModelCall | null;
 }
@@ -897,6 +907,7 @@ export interface GenerateConfig1 {
   logprobs: Logprobs;
   top_logprobs: TopLogprobs;
   parallel_tool_calls: ParallelToolCalls;
+  internal_tools: InternalTools;
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;

inspect_ai/_view/www/src/utils/Format.mjs

@@ -126,10 +126,17 @@ export const formatTime = (seconds) => {
     return `${seconds} sec`;
   } else if (seconds < 60 * 60) {
     return `${Math.floor(seconds / 60)} min ${seconds % 60} sec`;
+  } else if (seconds < 60 * 60 * 24) {
+    const hours = Math.floor(seconds / (60 * 60));
+    const minutes = Math.floor((seconds % (60 * 60)) / 60);
+    const remainingSeconds = seconds % 60;
+    return `${hours} hr ${minutes} min ${remainingSeconds} sec`;
   } else {
-    return `${Math.floor(seconds / (60 * 60 * 24))} days ${Math.floor(
-      seconds / 60,
-    )} min ${seconds % 60} sec`;
+    const days = Math.floor(seconds / (60 * 60 * 24));
+    const hours = Math.floor((seconds % (60 * 60 * 24)) / (60 * 60));
+    const minutes = Math.floor((seconds % (60 * 60)) / 60);
+    const remainingSeconds = seconds % 60;
+    return `${days} days ${hours} hr ${minutes} min ${remainingSeconds} sec`;
   }
 };
 

inspect_ai/_view/www/src/utils/{Json.mjs → json-worker.ts}

@@ -1,6 +1,6 @@
-// @ts-check
-
-export const asyncJsonParse = async (text) => {
+export const asyncJsonParse = async (text: string): Promise<any> => {
+  const encoder = new TextEncoder();
+  const encodedText = encoder.encode(text);
   const blob = new Blob([kWorkerCode], { type: "application/javascript" });
   const blobURL = URL.createObjectURL(blob);
   const worker = new Worker(blobURL);
@@ -17,7 +17,9 @@ export const asyncJsonParse = async (text) => {
         reject(new Error(error.message));
       };
     });
-    worker.postMessage({ scriptContent: kJson5ScriptBase64, text });
+    worker.postMessage({ scriptContent: kJson5ScriptBase64, encodedText }, [
+      encodedText.buffer,
+    ]);
     return await result;
   } finally {
     worker.terminate();
@@ -28,12 +30,14 @@ export const asyncJsonParse = async (text) => {
 const kWorkerCode = `
 self.onmessage = function (e) {
   eval(atob(e.data.scriptContent));
-  const text = e.data.text;
+  const { encodedText } = e.data;
+  const decoder = new TextDecoder();
+  const text = decoder.decode(encodedText);
   try {
-    const result = JSON5.parse(text);
-    self.postMessage({ success: true, result });
-  } catch (error) {
-    self.postMessage({ success: false, error: error.message });
+    const result = JSON.parse(text);
+    postMessage({ success: true, result });
+  } catch (err) {
+    postMessage({ success: false, error: err.message });
   }
 };`;
 

inspect_ai/_view/www/src/utils/vscode.ts

@@ -0,0 +1,36 @@
+/**
+ * Type definition for the VS Code API object
+ * Note: This is a minimal definition - expand based on your needs
+ */
+interface VSCodeApi {
+  postMessage(message: unknown): void;
+  getState(): unknown;
+  setState(state: unknown): void;
+}
+
+/**
+ * The cached instance of the VS Code API
+ */
+let vscodeApi: VSCodeApi | undefined;
+
+// Declare the acquireVsCodeApi function on the window object
+declare global {
+  interface Window {
+    acquireVsCodeApi?: () => VSCodeApi;
+  }
+}
+
+/**
+ * Gets or initializes the VS Code API instance
+ * @returns {VSCodeApi | undefined} The VS Code API instance if in VS Code environment, undefined otherwise
+ */
+export const getVscodeApi = (): VSCodeApi | undefined => {
+  if (window.acquireVsCodeApi) {
+    if (vscodeApi === undefined) {
+      vscodeApi = window.acquireVsCodeApi();
+    }
+    return vscodeApi;
+  } else {
+    return undefined;
+  }
+};

inspect_ai/_view/www/src/workspace/WorkSpace.mjs

@@ -43,7 +43,7 @@ import { debounce } from "../utils/sync.mjs";
  * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats for this eval
  * @param {import("../types/log").EvalResults} [props.evalResults] - The EvalResults for this eval
  * @param {import("../Types.mjs").CurrentLog} [props.log] - the current log
- * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples
+ * @param {import("../api/Types.ts").SampleSummary[]} [props.samples] - the samples
  * @param {import("../Types.mjs").SampleMode} props.sampleMode - the mode for displaying samples
  * @param {string} props.groupBy - what to group by
  * @param {string} props.groupByOrder - the grouping order
@@ -142,6 +142,8 @@ export const WorkSpace = ({
     }
   }, [divRef, task_id]);
 
+  const sampleTabScrollRef = useRef(/** @type {HTMLElement|null} */ (null));
+
   const resolvedTabs = useMemo(() => {
     // Tabs that are available within the app
     // Include the tab contents as well as any tools that the tab provides
@@ -154,6 +156,7 @@ export const WorkSpace = ({
       resolvedTabs.samples = {
         id: kEvalWorkspaceTabId,
         scrollable: samples.length === 1,
+        scrollRef: sampleTabScrollRef,
         label: samples?.length > 1 ? "Samples" : "Sample",
         content: () => {
           return html` <${SamplesTab}
@@ -178,6 +181,7 @@ export const WorkSpace = ({
             epoch=${epoch}
             sampleScrollPositionRef=${sampleScrollPositionRef}
             setSampleScrollPosition=${setSampleScrollPosition}
+            sampleTabScrollRef=${sampleTabScrollRef}
           />`;
         },
         tools: () => {
@@ -368,6 +372,7 @@ export const WorkSpace = ({
     evalResults=${evalResults}
     evalStats=${evalStats}
     samples=${samples}
+    evalDescriptor=${samplesDescriptor.evalDescriptor}
     status=${evalStatus}
     tabs=${resolvedTabs}
     selectedTab=${selectedTab}
@@ -386,6 +391,7 @@ const WorkspaceDisplay = ({
   evalResults,
   evalStats,
   samples,
+  evalDescriptor,
   status,
   showToggle,
   selectedTab,
@@ -442,6 +448,7 @@ const WorkspaceDisplay = ({
         onSelected=${onSelected}
         selected=${selectedTab === tab.id}
         scrollable=${!!tab.scrollable}
+        scrollRef=${tab.scrollRef}
         scrollPosition=${workspaceTabScrollPositionRef.current[tab.id]}
         setScrollPosition=${useCallback(
           (position) => {
@@ -456,20 +463,19 @@ const WorkspaceDisplay = ({
     }, [tabs]);
 
     return html`
-    
-    
     <${Navbar}
       evalSpec=${evalSpec}
       evalPlan=${evalPlan}
       evalResults=${evalResults}
       evalStats=${evalStats}
       samples=${samples}
+      evalDescriptor=${evalDescriptor}
       status=${status}
       file=${logFileName}
       showToggle=${showToggle}
-      
+
       offcanvas=${offcanvas}
-    />    
+    />
     <div ref=${divRef} class="workspace" style=${{
       paddingTop: "0rem",
       overflowY: "hidden",

inspect_ai/_view/www/vite.config.js

@@ -26,4 +26,11 @@ export default defineConfig({
       css: true,
     }),
   ],
+  resolve: {
+    alias: {
+      'react': 'preact/compat',
+      'react-dom': 'preact/compat',
+      'react-dom/test-utils': 'preact/test-utils'
+    }
+  }  
 });