inspect-ai 0.3.57__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/_view/www/log-schema.json

@@ -210,6 +210,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -281,6 +287,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -336,6 +348,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -429,6 +447,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -482,6 +506,36 @@
       "type": "object",
       "additionalProperties": false
     },
+    "ContentAudio": {
+      "properties": {
+        "type": {
+          "const": "audio",
+          "default": "audio",
+          "title": "Type",
+          "type": "string"
+        },
+        "audio": {
+          "title": "Audio",
+          "type": "string"
+        },
+        "format": {
+          "enum": [
+            "wav",
+            "mp3"
+          ],
+          "title": "Format",
+          "type": "string"
+        }
+      },
+      "required": [
+        "type",
+        "audio",
+        "format"
+      ],
+      "title": "ContentAudio",
+      "type": "object",
+      "additionalProperties": false
+    },
     "ContentImage": {
       "properties": {
         "type": {
@@ -535,6 +589,37 @@
       "type": "object",
       "additionalProperties": false
     },
+    "ContentVideo": {
+      "properties": {
+        "type": {
+          "const": "video",
+          "default": "video",
+          "title": "Type",
+          "type": "string"
+        },
+        "video": {
+          "title": "Video",
+          "type": "string"
+        },
+        "format": {
+          "enum": [
+            "mp4",
+            "mpeg",
+            "mov"
+          ],
+          "title": "Format",
+          "type": "string"
+        }
+      },
+      "required": [
+        "type",
+        "video",
+        "format"
+      ],
+      "title": "ContentVideo",
+      "type": "object",
+      "additionalProperties": false
+    },
     "ErrorEvent": {
       "description": "Event with sample error.",
       "properties": {
@@ -657,18 +742,6 @@
           "default": null,
           "title": "Epochs Reducer"
         },
-        "trace": {
-          "anyOf": [
-            {
-              "type": "boolean"
-            },
-            {
-              "type": "null"
-            }
-          ],
-          "default": null,
-          "title": "Trace"
-        },
         "approval": {
           "anyOf": [
             {
@@ -847,7 +920,6 @@
         "sample_id",
         "epochs",
         "epochs_reducer",
-        "trace",
         "approval",
         "fail_on_error",
         "message_limit",
@@ -1065,6 +1137,7 @@
             "logprobs": null,
             "top_logprobs": null,
             "parallel_tool_calls": null,
+            "internal_tools": null,
             "max_tool_output": null,
             "cache_prompt": null,
             "reasoning_effort": null
@@ -1444,7 +1517,8 @@
             "time",
             "message",
             "token",
-            "operator"
+            "operator",
+            "custom"
           ],
           "title": "Type",
           "type": "string"
@@ -2118,6 +2192,18 @@
           "default": null,
           "title": "Parallel Tool Calls"
         },
+        "internal_tools": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Internal Tools"
+        },
         "max_tool_output": {
           "anyOf": [
             {
@@ -2186,6 +2272,7 @@
         "logprobs",
         "top_logprobs",
         "parallel_tool_calls",
+        "internal_tools",
         "max_tool_output",
         "cache_prompt",
         "reasoning_effort"
@@ -2609,6 +2696,18 @@
         "output": {
           "$ref": "#/$defs/ModelOutput"
         },
+        "error": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Error"
+        },
         "cache": {
           "anyOf": [
             {
@@ -2647,6 +2746,7 @@
         "tool_choice",
         "config",
         "output",
+        "error",
         "cache",
         "call"
       ],
@@ -2994,7 +3094,8 @@
             "message",
             "time",
             "token",
-            "operator"
+            "operator",
+            "custom"
           ],
           "title": "Type",
           "type": "string"
@@ -3721,6 +3822,12 @@
             {
               "$ref": "#/$defs/ContentImage"
             },
+            {
+              "$ref": "#/$defs/ContentAudio"
+            },
+            {
+              "$ref": "#/$defs/ContentVideo"
+            },
             {
               "items": {
                 "anyOf": [
@@ -3729,6 +3836,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -4123,6 +4236,7 @@
           "best_of": null,
           "cache_prompt": null,
           "frequency_penalty": null,
+          "internal_tools": null,
           "logit_bias": null,
           "logprobs": null,
           "max_connections": null,

inspect_ai/_view/www/package.json

@@ -30,8 +30,10 @@
     "bootstrap": "^5.3.3",
     "bootstrap-icons": "^1.11.3",
     "clipboard": "^2.0.11",
+    "codemirror": "^6.0.1",
     "fast-json-patch": "^3.1.1",
     "fflate": "^0.8.2",
+    "filtrex": "^3.1.0",
     "htm": "^3.1.1",
     "json": "^11.0.0",
     "json5": "^2.2.3",

inspect_ai/_view/www/src/App.mjs

@@ -38,7 +38,7 @@ import {
 } from "./samples/SamplesDescriptor.mjs";
 import { byEpoch, bySample, sortSamples } from "./samples/tools/SortFilter.mjs";
 import { resolveAttachments } from "./utils/attachments.mjs";
-import { filterFnForType } from "./samples/tools/filters.mjs";
+import { filterSamples } from "./samples/tools/filters.mjs";
 
 import {
   kEvalWorkspaceTabId,
@@ -308,21 +308,19 @@ export function App({
 
   useEffect(() => {
     const samples = selectedLog?.contents?.sampleSummaries || [];
-    const filtered = samples.filter((sample) => {
+    const { result: prefiltered } = filterSamples(
+      evalDescriptor,
+      samples,
+      filter?.value,
+    );
+    const filtered = prefiltered.filter((sample) => {
       // Filter by epoch if specified
       if (epoch && epoch !== "all") {
         if (epoch !== sample.epoch + "") {
           return false;
         }
       }
-
-      // Apply the filter
-      const filterFn = filterFnForType(filter);
-      if (filterFn && filter.value) {
-        return filterFn(samplesDescriptor, sample, filter.value);
-      } else {
-        return true;
-      }
+      return true;
     });
 
     // Sort the samples

inspect_ai/_view/www/src/Types.mjs

@@ -25,7 +25,6 @@
 /**
  * @typedef {Object} ScoreFilter
  * @property {string} [value]
- * @property {string} [type]
  */
 
 /**

inspect_ai/_view/www/src/components/ChatView.mjs

@@ -7,6 +7,54 @@ import { MessageContent } from "./MessageContent.mjs";
 import { ExpandablePanel } from "./ExpandablePanel.mjs";
 import { FontSize, TextStyle } from "../appearance/Fonts.mjs";
 import { resolveToolInput, ToolCallView } from "./Tools.mjs";
+import { VirtualList } from "./VirtualList.mjs";
+
+/**
+ * Renders the ChatViewVirtualList component.
+ *
+ * @param {Object} props - The properties passed to the component.
+ * @param {string} props.id - The ID for the chat view.
+ * @param {import("../types/log").Messages} props.messages - The array of chat messages.
+ * @param {"compact" | "complete"} [props.toolCallStyle] - Whether to show tool calls
+ * @param {Object} [props.style] - Inline styles for the chat view.
+ * @param {boolean} props.indented - Whether the chatview has indented messages
+ * @param {boolean} [props.numbered] - Whether the chatview is numbered
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable parent element
+ * @returns {import("preact").JSX.Element} The component.
+ */
+export const ChatViewVirtualList = ({
+  id,
+  messages,
+  toolCallStyle,
+  style,
+  indented,
+  numbered = true,
+  scrollRef,
+}) => {
+  const collapsedMessages = resolveMessages(messages);
+
+  const renderRow = (item, index) => {
+    const number =
+      collapsedMessages.length > 1 && numbered ? index + 1 : undefined;
+    return html`<${ChatMessageRow}
+      id=${id}
+      number=${number}
+      resolvedMessage=${item}
+      indented=${indented}
+      toolCallStyle=${toolCallStyle}
+    />`;
+  };
+
+  const result = html`<${VirtualList}
+    data=${collapsedMessages}
+    tabIndex="0"
+    renderRow=${renderRow}
+    scrollRef=${scrollRef}
+    style=${{ width: "100%", marginTop: "1em", ...style }}
+  />`;
+
+  return result;
+};
 
 /**
  * Renders the ChatView component.
@@ -28,6 +76,90 @@ export const ChatView = ({
   indented,
   numbered = true,
 }) => {
+  const collapsedMessages = resolveMessages(messages);
+  const result = html` <div style=${style}>
+    ${collapsedMessages.map((msg, index) => {
+      const number =
+        collapsedMessages.length > 1 && numbered ? index + 1 : undefined;
+      return html`<${ChatMessageRow}
+        id=${id}
+        number=${number}
+        resolvedMessage=${msg}
+        indented=${indented}
+        toolCallStyle=${toolCallStyle}
+      />`;
+    })}
+  </div>`;
+  return result;
+};
+
+/**
+ * Renders the ChatMessage component.
+ *
+ * @param {Object} props - The properties passed to the component.
+ * @param {string} props.id - The ID for the chat view.
+ * @param {number} [props.number] - The message number
+ * @param {ResolvedMessage} props.resolvedMessage - The array of chat messages.
+ * @param {"compact" | "complete"} [props.toolCallStyle] - Whether to show tool calls
+ * @param {boolean} props.indented - Whether the chatview has indented messages
+ * @returns {import("preact").JSX.Element} The component.
+ */
+export const ChatMessageRow = ({
+  id,
+  number,
+  resolvedMessage,
+  toolCallStyle,
+  indented,
+}) => {
+  if (number) {
+    return html` <div
+      style=${{
+        display: "grid",
+        gridTemplateColumns: "max-content auto",
+        columnGap: "0.4em",
+      }}
+    >
+      <div
+        style=${{
+          fontSize: FontSize.smaller,
+          ...TextStyle.secondary,
+          marginTop: "0.1em",
+        }}
+      >
+        ${number}
+      </div>
+      <${ChatMessage}
+        id=${`${id}-chat-messages`}
+        message=${resolvedMessage.message}
+        toolMessages=${resolvedMessage.toolMessages}
+        indented=${indented}
+        toolCallStyle=${toolCallStyle}
+      />
+    </div>`;
+  } else {
+    return html`<${ChatMessage}
+      id=${`${id}-chat-messages`}
+      message=${resolvedMessage.message}
+      toolMessages=${resolvedMessage.toolMessages}
+      indented=${indented}
+      toolCallStyle=${toolCallStyle}
+    />`;
+  }
+};
+
+/**
+ * @typedef {Object} ResolvedMessage
+ * @property {import("../types/log").ChatMessageAssistant | import("../types/log").ChatMessageSystem | import("../types/log").ChatMessageUser} message - The main chat message.
+ * @property {import("../types/log").ChatMessageTool[]} [toolMessages] - Optional array of tool-related messages.
+ */
+
+/**
+ * Renders the ChatView component.
+ *
+ * @param {import("../types/log").Messages} messages - The array of chat messages.
+ * @returns {ResolvedMessage[]} The component.
+ */
+export const resolveMessages = (messages) => {
   // Filter tool messages into a sidelist that the chat stream
   // can use to lookup the tool responses
 
@@ -88,49 +220,7 @@ export const ChatView = ({
   if (systemMessage && systemMessage.content.length > 0) {
     collapsedMessages.unshift({ message: systemMessage });
   }
-
-  const result = html`
-    <div style=${style}>
-      ${collapsedMessages.map((msg, index) => {
-        if (collapsedMessages.length > 1 && numbered) {
-          return html` <div
-            style=${{
-              display: "grid",
-              gridTemplateColumns: "max-content auto",
-              columnGap: "0.4em",
-            }}
-          >
-            <div
-              style=${{
-                fontSize: FontSize.smaller,
-                ...TextStyle.secondary,
-                marginTop: "0.1em",
-              }}
-            >
-              ${index + 1}
-            </div>
-            <${ChatMessage}
-              id=${`${id}-chat-messages`}
-              message=${msg.message}
-              toolMessages=${msg.toolMessages}
-              indented=${indented}
-              toolCallStyle=${toolCallStyle}
-            />
-          </div>`;
-        } else {
-          return html` <${ChatMessage}
-            id=${`${id}-chat-messages`}
-            message=${msg.message}
-            toolMessages=${msg.toolMessages}
-            indented=${indented}
-            toolCallStyle=${toolCallStyle}
-          />`;
-        }
-      })}
-    </div>
-  `;
-
-  return result;
+  return collapsedMessages;
 };
 
 /**

inspect_ai/_view/www/src/components/ExpandablePanel.mjs

@@ -70,10 +70,6 @@ export const ExpandablePanel = ({
     contentsStyle.border = "solid var(--bs-light-border-subtle) 1px";
   }
 
-  if (!showToggle) {
-    contentsStyle.marginBottom = "1em";
-  }
-
   return html`<div
       class="expandable-panel"
       ref=${contentsRef}

inspect_ai/_view/www/src/components/LargeModal.mjs

@@ -5,25 +5,24 @@ import { FontSize } from "../appearance/Fonts.mjs";
 import { ProgressBar } from "./ProgressBar.mjs";
 import { MessageBand } from "./MessageBand.mjs";
 
-export const LargeModal = (props) => {
-  const {
-    id,
-    title,
-    detail,
-    detailTools,
-    footer,
-    onkeyup,
-    visible,
-    onHide,
-    showProgress,
-    children,
-    initialScrollPositionRef,
-    setInitialScrollPosition,
-    warning,
-    warningHidden,
-    setWarningHidden,
-  } = props;
-
+export const LargeModal = ({
+  id,
+  title,
+  detail,
+  detailTools,
+  footer,
+  onkeyup,
+  visible,
+  onHide,
+  showProgress,
+  children,
+  initialScrollPositionRef,
+  setInitialScrollPosition,
+  warning,
+  warningHidden,
+  setWarningHidden,
+  scrollRef,
+}) => {
   // The footer
   const modalFooter = footer
     ? html`<div class="modal-footer">${footer}</div>`
@@ -31,7 +30,7 @@ export const LargeModal = (props) => {
 
   // Support restoring the scroll position
   // but only do this for the first time that the children are set
-  const scrollRef = useRef(/** @type {HTMLElement|null} */ (null));
+  scrollRef = scrollRef || useRef(/** @type {HTMLElement|null} */ (null));
   useEffect(() => {
     if (scrollRef.current) {
       setTimeout(() => {

inspect_ai/_view/www/src/components/MessageBand.mjs

@@ -8,7 +8,7 @@ export const MessageBand = ({ message, hidden, setHidden, type }) => {
   const bgColor =
     type === "info" ? "var(--bs-light)" : "var(--bs-" + type + "-bg-subtle)";
   const color =
-    "var(--bs-" + (type === "info" ? "secondary" : type) + "-text-emphasis)";
+    type === "info" ? undefined : "var(--bs-" + type + "-text-emphasis)";
 
   return html`
     <div
@@ -32,7 +32,7 @@ export const MessageBand = ({ message, hidden, setHidden, type }) => {
           fontSize: FontSize["title-secondary"],
           margin: "0",
           padding: "0",
-          color: "var(--bs-" + type + "-text-emphasis)",
+          color: color,
           height: FontSize["title-secondary"],
           lineHeight: FontSize["title-secondary"],
         }}

inspect_ai/_view/www/src/components/MessageContent.mjs

@@ -7,7 +7,7 @@ import { ToolOutput } from "./Tools.mjs";
  * Supports rendering strings, images, and tools using specific renderers.
  *
  * @param {Object} props - The props object.
- * @param {string|string[]| (import("../types/log").ContentText | import("../types/log").ContentImage | import("../Types.mjs").ContentTool)[]} props.contents - The content or array of contents to render.
+ * @param {string|string[]| (import("../types/log").ContentText | import("../types/log").ContentImage | import("../types/log").ContentAudio | import("../types/log").ContentVideo | import("../Types.mjs").ContentTool)[]} props.contents - The content or array of contents to render.
  * @returns {import("preact").JSX.Element | import("preact").JSX.Element[]} The component.
  */
 export const MessageContent = ({ contents }) => {
@@ -61,9 +61,51 @@ const messageRenderers = {
       }
     },
   },
+  audio: {
+    render: (content) => {
+      return html` <audio controls>
+        <source
+          src=${content.audio}
+          type=${mimeTypeForFormat(content.format)}
+        />
+      </audio>`;
+    },
+  },
+  video: {
+    render: (content) => {
+      return html` <video width="500" height="375" controls>
+        <source
+          src=${content.video}
+          type=${mimeTypeForFormat(content.format)}
+        />
+      </video>`;
+    },
+  },
   tool: {
     render: (content) => {
       return html`<${ToolOutput} output=${content.content} />`;
     },
   },
 };
+
+/**
+ * Renders message content based on its type.
+ * Supports rendering strings, images, and tools using specific renderers.
+ *
+ * @param {import("../types/log").Format | import("../types/log").Format1 } format - The format
+ * @returns {string} - The mime type.
+ */
+const mimeTypeForFormat = (format) => {
+  switch (format) {
+    case "mov":
+      return "video/quicktime";
+    case "wav":
+      return "audio/wav";
+    case "mp3":
+      return "audio/mpeg";
+    case "mp4":
+      return "video/mp4";
+    case "mpeg":
+      return "video/mpeg";
+  }
+};

inspect_ai/_view/www/src/components/TabSet.mjs

@@ -38,13 +38,15 @@ export const TabPanel = ({
   selected,
   style,
   scrollable,
+  scrollRef,
   classes,
   scrollPosition,
   setScrollPosition,
   children,
 }) => {
   const tabContentsId = computeTabContentsId(id, index);
-  const tabContentsRef = useRef(/** @type {HTMLElement|null} */ (null));
+  const tabContentsRef =
+    scrollRef || useRef(/** @type {HTMLElement|null} */ (null));
   useEffect(() => {
     setTimeout(() => {
       if (