inspect-ai 0.3.57__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/_view/www/src/utils/Format.mjs

@@ -126,10 +126,17 @@ export const formatTime = (seconds) => {
     return `${seconds} sec`;
   } else if (seconds < 60 * 60) {
     return `${Math.floor(seconds / 60)} min ${seconds % 60} sec`;
+  } else if (seconds < 60 * 60 * 24) {
+    const hours = Math.floor(seconds / (60 * 60));
+    const minutes = Math.floor((seconds % (60 * 60)) / 60);
+    const remainingSeconds = seconds % 60;
+    return `${hours} hr ${minutes} min ${remainingSeconds} sec`;
   } else {
-    return `${Math.floor(seconds / (60 * 60 * 24))} days ${Math.floor(
-      seconds / 60,
-    )} min ${seconds % 60} sec`;
+    const days = Math.floor(seconds / (60 * 60 * 24));
+    const hours = Math.floor((seconds % (60 * 60 * 24)) / (60 * 60));
+    const minutes = Math.floor((seconds % (60 * 60)) / 60);
+    const remainingSeconds = seconds % 60;
+    return `${days} days ${hours} hr ${minutes} min ${remainingSeconds} sec`;
   }
 };
 

inspect_ai/_view/www/src/utils/Json.mjs

@@ -1,6 +1,8 @@
 // @ts-check
 
 export const asyncJsonParse = async (text) => {
+  const encoder = new TextEncoder();
+  const encodedText = encoder.encode(text);
   const blob = new Blob([kWorkerCode], { type: "application/javascript" });
   const blobURL = URL.createObjectURL(blob);
   const worker = new Worker(blobURL);
@@ -17,7 +19,9 @@ export const asyncJsonParse = async (text) => {
         reject(new Error(error.message));
       };
     });
-    worker.postMessage({ scriptContent: kJson5ScriptBase64, text });
+    worker.postMessage({ scriptContent: kJson5ScriptBase64, encodedText }, [
+      encodedText.buffer,
+    ]);
     return await result;
   } finally {
     worker.terminate();
@@ -28,12 +32,14 @@ export const asyncJsonParse = async (text) => {
 const kWorkerCode = `
 self.onmessage = function (e) {
   eval(atob(e.data.scriptContent));
-  const text = e.data.text;
+  const { encodedText } = e.data;
+  const decoder = new TextDecoder();
+  const text = decoder.decode(encodedText);
   try {
-    const result = JSON5.parse(text);
-    self.postMessage({ success: true, result });
-  } catch (error) {
-    self.postMessage({ success: false, error: error.message });
+    const result = JSON.parse(text);
+    postMessage({ success: true, result });
+  } catch (err) {
+    postMessage({ success: false, error: err.message });
   }
 };`;
 

inspect_ai/_view/www/src/workspace/WorkSpace.mjs

@@ -142,6 +142,8 @@ export const WorkSpace = ({
     }
   }, [divRef, task_id]);
 
+  const sampleTabScrollRef = useRef(/** @type {HTMLElement|null} */ (null));
+
   const resolvedTabs = useMemo(() => {
     // Tabs that are available within the app
     // Include the tab contents as well as any tools that the tab provides
@@ -154,6 +156,7 @@ export const WorkSpace = ({
       resolvedTabs.samples = {
         id: kEvalWorkspaceTabId,
         scrollable: samples.length === 1,
+        scrollRef: sampleTabScrollRef,
         label: samples?.length > 1 ? "Samples" : "Sample",
         content: () => {
           return html` <${SamplesTab}
@@ -178,6 +181,7 @@ export const WorkSpace = ({
             epoch=${epoch}
             sampleScrollPositionRef=${sampleScrollPositionRef}
             setSampleScrollPosition=${setSampleScrollPosition}
+            sampleTabScrollRef=${sampleTabScrollRef}
           />`;
         },
         tools: () => {
@@ -368,6 +372,7 @@ export const WorkSpace = ({
     evalResults=${evalResults}
     evalStats=${evalStats}
     samples=${samples}
+    evalDescriptor=${samplesDescriptor.evalDescriptor}
     status=${evalStatus}
     tabs=${resolvedTabs}
     selectedTab=${selectedTab}
@@ -386,6 +391,7 @@ const WorkspaceDisplay = ({
   evalResults,
   evalStats,
   samples,
+  evalDescriptor,
   status,
   showToggle,
   selectedTab,
@@ -442,6 +448,7 @@ const WorkspaceDisplay = ({
         onSelected=${onSelected}
         selected=${selectedTab === tab.id}
         scrollable=${!!tab.scrollable}
+        scrollRef=${tab.scrollRef}
         scrollPosition=${workspaceTabScrollPositionRef.current[tab.id]}
         setScrollPosition=${useCallback(
           (position) => {
@@ -456,20 +463,19 @@ const WorkspaceDisplay = ({
     }, [tabs]);
 
     return html`
-    
-    
     <${Navbar}
       evalSpec=${evalSpec}
       evalPlan=${evalPlan}
       evalResults=${evalResults}
       evalStats=${evalStats}
       samples=${samples}
+      evalDescriptor=${evalDescriptor}
       status=${status}
       file=${logFileName}
       showToggle=${showToggle}
-      
+
       offcanvas=${offcanvas}
-    />    
+    />
     <div ref=${divRef} class="workspace" style=${{
       paddingTop: "0rem",
       overflowY: "hidden",

inspect_ai/_view/www/vite.config.js

@@ -26,4 +26,11 @@ export default defineConfig({
       css: true,
     }),
   ],
+  resolve: {
+    alias: {
+      'react': 'preact/compat',
+      'react-dom': 'preact/compat',
+      'react-dom/test-utils': 'preact/test-utils'
+    }
+  }  
 });

inspect_ai/_view/www/yarn.lock

@@ -169,6 +169,70 @@
     "@babel/helper-validator-identifier" "^7.24.7"
     to-fast-properties "^2.0.0"
 
+"@codemirror/autocomplete@^6.0.0":
+  version "6.18.3"
+  resolved "https://registry.yarnpkg.com/@codemirror/autocomplete/-/autocomplete-6.18.3.tgz#f9ea79a2f369662516f71bc0b2f819454d3c8e00"
+  integrity sha512-1dNIOmiM0z4BIBwxmxEfA1yoxh1MF/6KPBbh20a5vphGV0ictKlgQsbJs6D6SkR6iJpGbpwRsa6PFMNlg9T9pQ==
+  dependencies:
+    "@codemirror/language" "^6.0.0"
+    "@codemirror/state" "^6.0.0"
+    "@codemirror/view" "^6.17.0"
+    "@lezer/common" "^1.0.0"
+
+"@codemirror/commands@^6.0.0":
+  version "6.7.1"
+  resolved "https://registry.yarnpkg.com/@codemirror/commands/-/commands-6.7.1.tgz#04561e95bc0779eaa49efd63e916c4efb3bbf6d6"
+  integrity sha512-llTrboQYw5H4THfhN4U3qCnSZ1SOJ60ohhz+SzU0ADGtwlc533DtklQP0vSFaQuCPDn3BPpOd1GbbnUtwNjsrw==
+  dependencies:
+    "@codemirror/language" "^6.0.0"
+    "@codemirror/state" "^6.4.0"
+    "@codemirror/view" "^6.27.0"
+    "@lezer/common" "^1.1.0"
+
+"@codemirror/language@^6.0.0":
+  version "6.10.6"
+  resolved "https://registry.yarnpkg.com/@codemirror/language/-/language-6.10.6.tgz#3770aa55fce575b45b1037b390b576907f0061c7"
+  integrity sha512-KrsbdCnxEztLVbB5PycWXFxas4EOyk/fPAfruSOnDDppevQgid2XZ+KbJ9u+fDikP/e7MW7HPBTvTb8JlZK9vA==
+  dependencies:
+    "@codemirror/state" "^6.0.0"
+    "@codemirror/view" "^6.23.0"
+    "@lezer/common" "^1.1.0"
+    "@lezer/highlight" "^1.0.0"
+    "@lezer/lr" "^1.0.0"
+    style-mod "^4.0.0"
+
+"@codemirror/lint@^6.0.0":
+  version "6.8.4"
+  resolved "https://registry.yarnpkg.com/@codemirror/lint/-/lint-6.8.4.tgz#7d8aa5d1a6dec89ffcc23ad45ddca2e12e90982d"
+  integrity sha512-u4q7PnZlJUojeRe8FJa/njJcMctISGgPQ4PnWsd9268R4ZTtU+tfFYmwkBvgcrK2+QQ8tYFVALVb5fVJykKc5A==
+  dependencies:
+    "@codemirror/state" "^6.0.0"
+    "@codemirror/view" "^6.35.0"
+    crelt "^1.0.5"
+
+"@codemirror/search@^6.0.0":
+  version "6.5.8"
+  resolved "https://registry.yarnpkg.com/@codemirror/search/-/search-6.5.8.tgz#b59b3659b46184cc75d6108d7c050a4ca344c3a0"
+  integrity sha512-PoWtZvo7c1XFeZWmmyaOp2G0XVbOnm+fJzvghqGAktBW3cufwJUWvSCcNG0ppXiBEM05mZu6RhMtXPv2hpllig==
+  dependencies:
+    "@codemirror/state" "^6.0.0"
+    "@codemirror/view" "^6.0.0"
+    crelt "^1.0.5"
+
+"@codemirror/state@^6.0.0", "@codemirror/state@^6.4.0":
+  version "6.4.1"
+  resolved "https://registry.yarnpkg.com/@codemirror/state/-/state-6.4.1.tgz#da57143695c056d9a3c38705ed34136e2b68171b"
+  integrity sha512-QkEyUiLhsJoZkbumGZlswmAhA7CBU02Wrz7zvH4SrcifbsqwlXShVXg65f3v/ts57W3dqyamEriMhij1Z3Zz4A==
+
+"@codemirror/view@^6.0.0", "@codemirror/view@^6.17.0", "@codemirror/view@^6.23.0", "@codemirror/view@^6.27.0", "@codemirror/view@^6.35.0":
+  version "6.35.0"
+  resolved "https://registry.yarnpkg.com/@codemirror/view/-/view-6.35.0.tgz#890e8e31a58edf65cdf193049fe9f3fdec20cc82"
+  integrity sha512-I0tYy63q5XkaWsJ8QRv5h6ves7kvtrBWjBcnf/bzohFJQc5c14a1AQRdE8QpPF9eMp5Mq2FMm59TCj1gDfE7kw==
+  dependencies:
+    "@codemirror/state" "^6.4.0"
+    style-mod "^4.1.0"
+    w3c-keyname "^2.2.4"
+
 "@esbuild/aix-ppc64@0.21.5":
   version "0.21.5"
   resolved "https://registry.yarnpkg.com/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz#c7184a326533fcdf1b8ee0733e21c713b975575f"
@@ -372,6 +436,25 @@
     "@jridgewell/resolve-uri" "^3.1.0"
     "@jridgewell/sourcemap-codec" "^1.4.14"
 
+"@lezer/common@^1.0.0", "@lezer/common@^1.1.0":
+  version "1.2.3"
+  resolved "https://registry.yarnpkg.com/@lezer/common/-/common-1.2.3.tgz#138fcddab157d83da557554851017c6c1e5667fd"
+  integrity sha512-w7ojc8ejBqr2REPsWxJjrMFsA/ysDCFICn8zEOR9mrqzOu2amhITYuLD8ag6XZf0CFXDrhKqw7+tW8cX66NaDA==
+
+"@lezer/highlight@^1.0.0":
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/@lezer/highlight/-/highlight-1.2.1.tgz#596fa8f9aeb58a608be0a563e960c373cbf23f8b"
+  integrity sha512-Z5duk4RN/3zuVO7Jq0pGLJ3qynpxUVsh7IbUbGj88+uV2ApSAn6kWg2au3iJb+0Zi7kKtqffIESgNcRXWZWmSA==
+  dependencies:
+    "@lezer/common" "^1.0.0"
+
+"@lezer/lr@^1.0.0":
+  version "1.4.2"
+  resolved "https://registry.yarnpkg.com/@lezer/lr/-/lr-1.4.2.tgz#931ea3dea8e9de84e90781001dae30dea9ff1727"
+  integrity sha512-pu0K1jCIdnQ12aWNaAVU5bzi7Bd1w54J3ECgANPmYLtQKP0HBj2cE/5coBD66MT10xbtIuUr7tg0Shbsvk0mDA==
+  dependencies:
+    "@lezer/common" "^1.0.0"
+
 "@nodelib/fs.scandir@2.1.5":
   version "2.1.5"
   resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5"
@@ -619,6 +702,19 @@ clipboard@^2.0.11:
     select "^1.1.2"
     tiny-emitter "^2.0.0"
 
+codemirror@^6.0.1:
+  version "6.0.1"
+  resolved "https://registry.yarnpkg.com/codemirror/-/codemirror-6.0.1.tgz#62b91142d45904547ee3e0e0e4c1a79158035a29"
+  integrity sha512-J8j+nZ+CdWmIeFIGXEFbFPtpiYacFMDR8GlHK3IyHQJMCaVRfGx9NT+Hxivv1ckLWPvNdZqndbr/7lVhrf/Svg==
+  dependencies:
+    "@codemirror/autocomplete" "^6.0.0"
+    "@codemirror/commands" "^6.0.0"
+    "@codemirror/language" "^6.0.0"
+    "@codemirror/lint" "^6.0.0"
+    "@codemirror/search" "^6.0.0"
+    "@codemirror/state" "^6.0.0"
+    "@codemirror/view" "^6.0.0"
+
 color-convert@^1.9.0:
   version "1.9.3"
   resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
@@ -653,6 +749,11 @@ convert-source-map@^2.0.0:
   resolved "https://registry.yarnpkg.com/convert-source-map/-/convert-source-map-2.0.0.tgz#4b560f649fc4e918dd0ab75cf4961e8bc882d82a"
   integrity sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==
 
+crelt@^1.0.5:
+  version "1.0.6"
+  resolved "https://registry.yarnpkg.com/crelt/-/crelt-1.0.6.tgz#7cc898ea74e190fb6ef9dae57f8f81cf7302df72"
+  integrity sha512-VQ2MBenTq1fWZUH9DJNGti7kKv6EeAuYr3cLwxUWhIu1baTaXh4Ib5W2CqHVqib4/MqbYGJqiL3Zb8GJZr3l4g==
+
 cross-spawn@^7.0.2:
   version "7.0.3"
   resolved "https://registry.yarnpkg.com/cross-spawn/-/cross-spawn-7.0.3.tgz#f73a85b9d5d41d045551c177e2882d4ac85728a6"
@@ -885,6 +986,11 @@ file-entry-cache@^8.0.0:
   dependencies:
     flat-cache "^4.0.0"
 
+filtrex@^3.1.0:
+  version "3.1.0"
+  resolved "https://registry.yarnpkg.com/filtrex/-/filtrex-3.1.0.tgz#5ec00994615ff10e5e09c89bb290c855cb408c21"
+  integrity sha512-mHzZ2wUISETF1OaEcNRiGz1ljuIV8c/C9td9qyAZ+wTwigkAk5RO9YrCxQKk5H9v7joDRFIBik9U5RTK9eXZ/A==
+
 find-up@^5.0.0:
   version "5.0.0"
   resolved "https://registry.yarnpkg.com/find-up/-/find-up-5.0.0.tgz#4c92819ecb7083561e4f4a240a86be5198f536fc"
@@ -1367,6 +1473,11 @@ strip-json-comments@^3.1.1:
   resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-3.1.1.tgz#31f1281b3832630434831c310c01cccda8cbe006"
   integrity sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==
 
+style-mod@^4.0.0, style-mod@^4.1.0:
+  version "4.1.2"
+  resolved "https://registry.yarnpkg.com/style-mod/-/style-mod-4.1.2.tgz#ca238a1ad4786520f7515a8539d5a63691d7bf67"
+  integrity sha512-wnD1HyVqpJUI2+eKZ+eo1UwghftP6yuFheBqqe+bWCotBjC2K1YnteJILRMs3SM4V/0dLEW1SC27MWP5y+mwmw==
+
 supports-color@^5.3.0:
   version "5.5.0"
   resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-5.5.0.tgz#e2e69a44ac8772f78a1ec0b35b689df6530efc8f"
@@ -1442,6 +1553,11 @@ vite@^5.3.2:
   optionalDependencies:
     fsevents "~2.3.3"
 
+w3c-keyname@^2.2.4:
+  version "2.2.8"
+  resolved "https://registry.yarnpkg.com/w3c-keyname/-/w3c-keyname-2.2.8.tgz#7b17c8c6883d4e8b86ac8aba79d39e880f8869c5"
+  integrity sha512-dpojBhNsCNN7T82Tm7k26A6G9ML3NkhDsnw9n/eoxSRlVBB4CEtIQ/KTCLI2Fwf3ataSXRhYFkQi3SlnFwPvPQ==
+
 which@^2.0.1:
   version "2.0.2"
   resolved "https://registry.yarnpkg.com/which/-/which-2.0.2.tgz#7c6a8dd0a636a0327e10b59c9286eee93f3f51b1"

inspect_ai/approval/_human/util.py

@@ -5,7 +5,7 @@ from rich.text import Text
 
 from inspect_ai._util.transcript import transcript_markdown
 from inspect_ai.tool._tool_call import ToolCallContent, ToolCallView
-from inspect_ai.util._trace import trace_enabled
+from inspect_ai.util._display import display_type
 
 HUMAN_APPROVED = "Human operator approved tool call."
 HUMAN_REJECTED = "Human operator rejected the tool call."
@@ -18,7 +18,7 @@ def render_tool_approval(message: str, view: ToolCallView) -> list[RenderableTyp
     text_highlighter = ReprHighlighter()
 
     # ignore content if trace enabled
-    message = message.strip() if not trace_enabled() else ""
+    message = message.strip() if display_type() != "conversation" else ""
 
     def add_view_content(view_content: ToolCallContent) -> None:
         if view_content.title:

inspect_ai/approval/_policy.py

@@ -1,13 +1,13 @@
 import fnmatch
-import re
+import sys
 from dataclasses import dataclass
 from pathlib import Path
-from re import Pattern
 from typing import Any, Generator, cast
 
 from pydantic import BaseModel, Field, model_validator
 
 from inspect_ai._util.config import read_config_object
+from inspect_ai._util.format import format_function_call
 from inspect_ai._util.registry import registry_create, registry_lookup
 from inspect_ai.solver._task_state import TaskState
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
@@ -30,17 +30,23 @@ def policy_approver(policies: str | list[ApprovalPolicy]) -> Approver:
         policies = approval_policies_from_config(policies)
 
     # compile policy into approvers and regexes for matching
-    policy_matchers: list[tuple[list[Pattern[str]], Approver]] = []
+    policy_matchers: list[tuple[list[str], Approver]] = []
     for policy in policies:
         tools = [policy.tools] if isinstance(policy.tools, str) else policy.tools
-        patterns = [re.compile(fnmatch.translate(tool)) for tool in tools]
-        policy_matchers.append((patterns, policy.approver))
+        globs = [f"{tool}*" for tool in tools]
+        policy_matchers.append((globs, policy.approver))
 
     # generator for policies that match a tool_call
     def tool_approvers(tool_call: ToolCall) -> Generator[Approver, None, None]:
         for policy_matcher in iter(policy_matchers):
+            function_call = format_function_call(
+                tool_call.function, tool_call.arguments, width=sys.maxsize
+            )
             if any(
-                [pattern.match(tool_call.function) for pattern in policy_matcher[0]]
+                [
+                    fnmatch.fnmatch(function_call, pattern)
+                    for pattern in policy_matcher[0]
+                ]
             ):
                 yield policy_matcher[1]
 

inspect_ai/dataset/_sources/csv.py

@@ -1,4 +1,5 @@
 import csv
+import os
 from io import TextIOWrapper
 from pathlib import Path
 from typing import Any
@@ -75,7 +76,7 @@ def csv_dataset(
         dataset = MemoryDataset(
             samples=data_to_samples(valid_data, data_to_sample, auto_id),
             name=name,
-            location=csv_file,
+            location=os.path.abspath(csv_file),
         )
 
         # resolve relative file paths

inspect_ai/dataset/_sources/json.py

@@ -1,4 +1,5 @@
 import json
+import os
 from io import TextIOWrapper
 from pathlib import Path
 from typing import Any, cast
@@ -75,7 +76,7 @@ def json_dataset(
         dataset = MemoryDataset(
             samples=data_to_samples(dataset_reader(f), data_to_sample, auto_id),
             name=name,
-            location=json_file,
+            location=os.path.abspath(json_file),
         )
 
         # resolve relative file paths

inspect_ai/dataset/_sources/util.py

@@ -1,6 +1,6 @@
 from typing import Callable
 
-from inspect_ai._util.content import Content, ContentImage
+from inspect_ai._util.content import Content, ContentAudio, ContentImage, ContentVideo
 from inspect_ai._util.file import filesystem
 from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
@@ -44,24 +44,28 @@ def resolve_sample_files(dataset: Dataset) -> None:
             for path in sample.files.keys():
                 sample.files[path] = resolve_file(sample.files[path])
 
+        # check for setup script
+        if sample.setup is not None:
+            sample.setup = resolve_file(sample.setup)
+
         # check for image paths
         if not isinstance(sample.input, str):
-            sample.input = messages_with_resolved_images(sample.input, resolve_file)
+            sample.input = messages_with_resolved_content(sample.input, resolve_file)
 
 
-def messages_with_resolved_images(
+def messages_with_resolved_content(
     messages: list[ChatMessage], resolver: Callable[[str], str]
 ) -> list[ChatMessage]:
-    return [message_with_resolved_image(message, resolver) for message in messages]
+    return [message_with_resolved_content(message, resolver) for message in messages]
 
 
-def message_with_resolved_image(
+def message_with_resolved_content(
     message: ChatMessage, resolver: Callable[[str], str]
 ) -> ChatMessage:
     if isinstance(message, ChatMessageUser) and not isinstance(message.content, str):
         return ChatMessageUser(
             content=[
-                chat_content_with_resolved_image(content, resolver)
+                chat_content_with_resolved_content(content, resolver)
                 for content in message.content
             ],
             source=message.source,
@@ -70,7 +74,7 @@ def message_with_resolved_image(
         return message
 
 
-def chat_content_with_resolved_image(
+def chat_content_with_resolved_content(
     content: Content, resolver: Callable[[str], str]
 ) -> Content:
     if isinstance(content, ContentImage):
@@ -78,5 +82,9 @@ def chat_content_with_resolved_image(
             image=resolver(content.image),
             detail=content.detail,
         )
+    elif isinstance(content, ContentAudio):
+        return ContentAudio(audio=resolver(content.audio), format=content.format)
+    elif isinstance(content, ContentVideo):
+        return ContentVideo(video=resolver(content.video), format=content.format)
     else:
         return content

inspect_ai/log/_condense.py

@@ -6,7 +6,13 @@ from typing import (
 from pydantic import JsonValue
 
 from inspect_ai._util.constants import BASE_64_DATA_REMOVED
-from inspect_ai._util.content import Content, ContentImage, ContentText
+from inspect_ai._util.content import (
+    Content,
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.hash import mm3_hash
 from inspect_ai._util.json import JsonChange
 from inspect_ai._util.url import is_data_uri
@@ -304,3 +310,7 @@ def walk_content(content: Content, content_fn: Callable[[str], str]) -> Content:
         return content.model_copy(update=dict(text=content_fn(content.text)))
     elif isinstance(content, ContentImage):
         return content.model_copy(update=dict(image=content_fn(content.image)))
+    elif isinstance(content, ContentAudio):
+        return content.model_copy(update=dict(audio=content_fn(content.audio)))
+    elif isinstance(content, ContentVideo):
+        return content.model_copy(update=dict(video=content_fn(content.video)))

inspect_ai/log/_log.py

@@ -48,9 +48,6 @@ class EvalConfig(BaseModel):
     epochs_reducer: list[str] | None = Field(default=None)
     """Reducers for aggregating per-sample scores."""
 
-    trace: bool | None = Field(default=None)
-    """Trace message interactions with evaluated model to terminal."""
-
     approval: ApprovalPolicyConfig | None = Field(default=None)
     """Approval policy for tool use."""
 
@@ -117,7 +114,7 @@ class EvalConfig(BaseModel):
 
 
 class EvalSampleLimit(BaseModel):
-    type: Literal["context", "time", "message", "token", "operator"]
+    type: Literal["context", "time", "message", "token", "operator", "custom"]
     """The type of limit"""
 
     limit: int
@@ -355,7 +352,7 @@ class EvalResults(BaseModel):
         """Scorer used to compute results (deprecated)."""
         warn_once(
             logger,
-            "The 'scorer' field is deprecated. Use 'scorers' instead.",
+            "The 'scorer' field is deprecated. Use 'scores' instead.",
         )
         return self.scores[0] if self.scores else None
 
@@ -364,7 +361,7 @@ class EvalResults(BaseModel):
         """Metrics computed (deprecated)."""
         warn_once(
             logger,
-            "The 'metrics' field is deprecated. Access metrics through 'scorers' instead.",
+            "The 'metrics' field is deprecated. Access metrics through 'scores' instead.",
         )
         return self.scores[0].metrics if self.scores else {}
 

inspect_ai/log/_recorders/eval.py

@@ -13,7 +13,12 @@ from pydantic_core import to_json
 from typing_extensions import override
 
 from inspect_ai._util.constants import LOG_SCHEMA_VERSION
-from inspect_ai._util.content import ContentImage, ContentText
+from inspect_ai._util.content import (
+    ContentAudio,
+    ContentImage,
+    ContentText,
+    ContentVideo,
+)
 from inspect_ai._util.error import EvalError
 from inspect_ai._util.file import FileSystem, async_fileystem, dirname, file, filesystem
 from inspect_ai._util.json import jsonable_python
@@ -90,9 +95,11 @@ class EvalRecorder(FileRecorder):
         self.data: dict[str, ZipLogFile] = {}
 
     @override
-    async def log_init(self, eval: EvalSpec, location: str | None = None) -> str:
+    async def log_init(
+        self, eval: EvalSpec, location: str | None = None, *, clean: bool = False
+    ) -> str:
         # if the file exists then read summaries
-        if location is not None and self.fs.exists(location):
+        if not clean and location is not None and self.fs.exists(location):
             with file(location, "rb") as f:
                 with ZipFile(f, "r") as zip:
                     log_start = _read_start(zip)
@@ -229,7 +236,7 @@ class EvalRecorder(FileRecorder):
     async def write_log(cls, location: str, log: EvalLog) -> None:
         # write using the recorder (so we get all of the extra streams)
         recorder = EvalRecorder(dirname(location))
-        await recorder.log_init(log.eval, location)
+        await recorder.log_init(log.eval, location, clean=True)
         await recorder.log_start(log.eval, log.plan)
         for sample in log.samples or []:
             await recorder.log_sample(log.eval, sample)
@@ -244,12 +251,16 @@ def text_inputs(inputs: str | list[ChatMessage]) -> str | list[ChatMessage]:
         input: list[ChatMessage] = []
         for message in inputs:
             if not isinstance(message.content, str):
-                filtered_content: list[ContentText | ContentImage] = []
+                filtered_content: list[
+                    ContentText | ContentImage | ContentAudio | ContentVideo
+                ] = []
                 for content in message.content:
-                    if content.type != "image":
+                    if content.type == "text":
                         filtered_content.append(content)
-                if len(filtered_content) == 0:
-                    filtered_content.append(ContentText(text="(Image)"))
+                    else:
+                        filtered_content.append(
+                            ContentText(text=f"({content.type.capitalize()})")
+                        )
                 message.content = filtered_content
                 input.append(message)
             else:

inspect_ai/log/_samples.py

@@ -29,7 +29,7 @@ class ActiveSample:
         sandboxes: dict[str, SandboxConnection],
     ) -> None:
         self.id = uuid()
-        self.started = datetime.now().timestamp()
+        self.started: float | None = None
         self.completed: float | None = None
         self.task = task
         self.model = model
@@ -48,10 +48,15 @@ class ActiveSample:
 
     @property
     def execution_time(self) -> float:
-        completed = (
-            self.completed if self.completed is not None else datetime.now().timestamp()
-        )
-        return completed - self.started
+        if self.started is not None:
+            completed = (
+                self.completed
+                if self.completed is not None
+                else datetime.now().timestamp()
+            )
+            return completed - self.started
+        else:
+            return 0
 
     def interrupt(self, action: Literal["score", "error"]) -> None:
         self._interrupt_action = action
@@ -108,6 +113,14 @@ def sample_active() -> ActiveSample | None:
     return _sample_active.get(None)
 
 
+def active_sample_token_limit() -> int | None:
+    active = sample_active()
+    if active:
+        return active.token_limit
+    else:
+        return None
+
+
 def set_active_sample_token_limit(token_limit: int | None) -> None:
     active = sample_active()
     if active:
@@ -120,6 +133,14 @@ def set_active_sample_total_tokens(total_tokens: int) -> None:
         active.total_tokens = total_tokens
 
 
+def active_sample_message_limit() -> int | None:
+    active = sample_active()
+    if active:
+        return active.message_limit
+    else:
+        return None
+
+
 def set_active_sample_message_limit(message_limit: int | None) -> None:
     active = sample_active()
     if active: