inspect-ai 0.3.57__py3-none-any.whl → 0.3.58__py3-none-any.whl

inspect_ai/_util/kvstore.py

@@ -0,0 +1,73 @@
+import sqlite3
+from contextlib import AbstractContextManager
+from typing import Any, Optional, cast
+
+from .appdirs import inspect_data_dir
+
+
+class KVStore(AbstractContextManager["KVStore"]):
+    def __init__(self, filename: str, max_entries: int | None = None):
+        self.filename = filename
+        self.max_entries = max_entries
+
+    def __enter__(self) -> "KVStore":
+        self.conn = sqlite3.connect(self.filename)
+        self.conn.execute("""
+            CREATE TABLE IF NOT EXISTS kv_store (
+                key TEXT PRIMARY KEY,
+                value TEXT,
+                created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+            )
+        """)
+        self.conn.commit()
+        return self
+
+    def __exit__(self, *excinfo: Any) -> None:
+        self.conn.close()
+
+    def put(self, key: str, value: str) -> None:
+        # Insert or update the value
+        self.conn.execute(
+            """
+            INSERT OR REPLACE INTO kv_store (key, value, created_at)
+            VALUES (?, ?, CURRENT_TIMESTAMP)
+        """,
+            (key, value),
+        )
+
+        # If we have a max_entries limit, remove oldest entries
+        if self.max_entries:
+            count = self.count()
+            if count > self.max_entries:
+                self.conn.execute(
+                    """
+                    DELETE FROM kv_store
+                    WHERE key IN (
+                        SELECT key FROM kv_store
+                        ORDER BY created_at ASC
+                        LIMIT ?
+                    )
+                    """,
+                    (max(0, count - self.max_entries),),
+                )
+
+        self.conn.commit()
+
+    def get(self, key: str) -> Optional[str]:
+        cursor = self.conn.execute("SELECT value FROM kv_store WHERE key = ?", (key,))
+        result = cursor.fetchone()
+        return result[0] if result else None
+
+    def delete(self, key: str) -> bool:
+        cursor = self.conn.execute("DELETE FROM kv_store WHERE key = ?", (key,))
+        self.conn.commit()
+        return cursor.rowcount > 0
+
+    def count(self) -> int:
+        cursor = self.conn.execute("SELECT COUNT(*) FROM kv_store")
+        return cast(int, cursor.fetchone()[0])
+
+
+def inspect_kvstore(name: str, max_entries: int | None = None) -> KVStore:
+    filename = inspect_data_dir("kvstore") / f"{name}.db"
+    return KVStore(filename.as_posix(), max_entries=max_entries)

inspect_ai/_util/notgiven.py

@@ -0,0 +1,18 @@
+# Sentinel class used until PEP 0661 is accepted
+from typing import Literal
+
+from typing_extensions import override
+
+
+class NotGiven:
+    """A sentinel singleton class used to distinguish omitted keyword arguments from those passed in with the value None (which may have different behavior)."""
+
+    def __bool__(self) -> Literal[False]:
+        return False
+
+    @override
+    def __repr__(self) -> str:
+        return "NOT_GIVEN"
+
+
+NOT_GIVEN = NotGiven()

inspect_ai/_util/thread.py

@@ -0,0 +1,5 @@
+import threading
+
+
+def is_main_thread() -> bool:
+    return threading.current_thread() is threading.main_thread()

inspect_ai/_view/www/dist/assets/index.js

@@ -15547,12 +15547,46 @@ var require_assets = __commonJS({
           }
         }
       },
+      audio: {
+        render: (content) => {
+          return m$1` <audio controls>
+        <source
+          src=${content.audio}
+          type=${mimeTypeForFormat(content.format)}
+        />
+      </audio>`;
+        }
+      },
+      video: {
+        render: (content) => {
+          return m$1` <video width="500" height="375" controls>
+        <source
+          src=${content.video}
+          type=${mimeTypeForFormat(content.format)}
+        />
+      </video>`;
+        }
+      },
       tool: {
         render: (content) => {
           return m$1`<${ToolOutput} output=${content.content} />`;
         }
       }
     };
+    const mimeTypeForFormat = (format2) => {
+      switch (format2) {
+        case "mov":
+          return "video/quicktime";
+        case "wav":
+          return "audio/wav";
+        case "mp3":
+          return "audio/mpeg";
+        case "mp4":
+          return "video/mp4";
+        case "mpeg":
+          return "video/mpeg";
+      }
+    };
     const ChatView = ({
       id,
       messages,
@@ -16493,7 +16527,7 @@ ${entry.value}</pre
     };
     const MessageBand = ({ message, hidden, setHidden, type }) => {
       const bgColor = type === "info" ? "var(--bs-light)" : "var(--bs-" + type + "-bg-subtle)";
-      const color = "var(--bs-" + (type === "info" ? "secondary" : type) + "-text-emphasis)";
+      const color = type === "info" ? void 0 : "var(--bs-" + type + "-text-emphasis)";
       return m$1`
     <div
       style=${{
@@ -16516,7 +16550,7 @@ ${entry.value}</pre
         fontSize: FontSize["title-secondary"],
         margin: "0",
         padding: "0",
-        color: "var(--bs-" + type + "-text-emphasis)",
+        color,
         height: FontSize["title-secondary"],
         lineHeight: FontSize["title-secondary"]
       }}
@@ -30728,7 +30762,7 @@ self.onmessage = function (e) {
           var _a2;
           const text2 = inputString(current.input).join(" ");
           const scoreValue = evalDescriptor.score(current, selectedScore).value;
-          const scoreText = scoreValue ? String(scoreValue) : "";
+          const scoreText = scoreValue ? String(scoreValue) : current.error ? String(current.error) : "";
           previous[0] = Math.min(Math.max(previous[0], text2.length), 300);
           previous[1] = Math.min(
             Math.max(previous[1], arrayToString(current.target).length),

inspect_ai/_view/www/log-schema.json

@@ -210,6 +210,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -281,6 +287,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -336,6 +348,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -429,6 +447,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -482,6 +506,36 @@
       "type": "object",
       "additionalProperties": false
     },
+    "ContentAudio": {
+      "properties": {
+        "type": {
+          "const": "audio",
+          "default": "audio",
+          "title": "Type",
+          "type": "string"
+        },
+        "audio": {
+          "title": "Audio",
+          "type": "string"
+        },
+        "format": {
+          "enum": [
+            "wav",
+            "mp3"
+          ],
+          "title": "Format",
+          "type": "string"
+        }
+      },
+      "required": [
+        "type",
+        "audio",
+        "format"
+      ],
+      "title": "ContentAudio",
+      "type": "object",
+      "additionalProperties": false
+    },
     "ContentImage": {
       "properties": {
         "type": {
@@ -535,6 +589,37 @@
       "type": "object",
       "additionalProperties": false
     },
+    "ContentVideo": {
+      "properties": {
+        "type": {
+          "const": "video",
+          "default": "video",
+          "title": "Type",
+          "type": "string"
+        },
+        "video": {
+          "title": "Video",
+          "type": "string"
+        },
+        "format": {
+          "enum": [
+            "mp4",
+            "mpeg",
+            "mov"
+          ],
+          "title": "Format",
+          "type": "string"
+        }
+      },
+      "required": [
+        "type",
+        "video",
+        "format"
+      ],
+      "title": "ContentVideo",
+      "type": "object",
+      "additionalProperties": false
+    },
     "ErrorEvent": {
       "description": "Event with sample error.",
       "properties": {
@@ -657,18 +742,6 @@
           "default": null,
           "title": "Epochs Reducer"
         },
-        "trace": {
-          "anyOf": [
-            {
-              "type": "boolean"
-            },
-            {
-              "type": "null"
-            }
-          ],
-          "default": null,
-          "title": "Trace"
-        },
         "approval": {
           "anyOf": [
             {
@@ -847,7 +920,6 @@
         "sample_id",
         "epochs",
         "epochs_reducer",
-        "trace",
         "approval",
         "fail_on_error",
         "message_limit",
@@ -3721,6 +3793,12 @@
             {
               "$ref": "#/$defs/ContentImage"
             },
+            {
+              "$ref": "#/$defs/ContentAudio"
+            },
+            {
+              "$ref": "#/$defs/ContentVideo"
+            },
             {
               "items": {
                 "anyOf": [
@@ -3729,6 +3807,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },

inspect_ai/_view/www/src/components/MessageBand.mjs

@@ -8,7 +8,7 @@ export const MessageBand = ({ message, hidden, setHidden, type }) => {
   const bgColor =
     type === "info" ? "var(--bs-light)" : "var(--bs-" + type + "-bg-subtle)";
   const color =
-    "var(--bs-" + (type === "info" ? "secondary" : type) + "-text-emphasis)";
+    type === "info" ? undefined : "var(--bs-" + type + "-text-emphasis)";
 
   return html`
     <div
@@ -32,7 +32,7 @@ export const MessageBand = ({ message, hidden, setHidden, type }) => {
           fontSize: FontSize["title-secondary"],
           margin: "0",
           padding: "0",
-          color: "var(--bs-" + type + "-text-emphasis)",
+          color: color,
           height: FontSize["title-secondary"],
           lineHeight: FontSize["title-secondary"],
         }}

inspect_ai/_view/www/src/components/MessageContent.mjs

@@ -7,7 +7,7 @@ import { ToolOutput } from "./Tools.mjs";
  * Supports rendering strings, images, and tools using specific renderers.
  *
  * @param {Object} props - The props object.
- * @param {string|string[]| (import("../types/log").ContentText | import("../types/log").ContentImage | import("../Types.mjs").ContentTool)[]} props.contents - The content or array of contents to render.
+ * @param {string|string[]| (import("../types/log").ContentText | import("../types/log").ContentImage | import("../types/log").ContentAudio | import("../types/log").ContentVideo | import("../Types.mjs").ContentTool)[]} props.contents - The content or array of contents to render.
  * @returns {import("preact").JSX.Element | import("preact").JSX.Element[]} The component.
  */
 export const MessageContent = ({ contents }) => {
@@ -61,9 +61,51 @@ const messageRenderers = {
       }
     },
   },
+  audio: {
+    render: (content) => {
+      return html` <audio controls>
+        <source
+          src=${content.audio}
+          type=${mimeTypeForFormat(content.format)}
+        />
+      </audio>`;
+    },
+  },
+  video: {
+    render: (content) => {
+      return html` <video width="500" height="375" controls>
+        <source
+          src=${content.video}
+          type=${mimeTypeForFormat(content.format)}
+        />
+      </video>`;
+    },
+  },
   tool: {
     render: (content) => {
       return html`<${ToolOutput} output=${content.content} />`;
     },
   },
 };
+
+/**
+ * Renders message content based on its type.
+ * Supports rendering strings, images, and tools using specific renderers.
+ *
+ * @param {import("../types/log").Format | import("../types/log").Format1 } format - The format
+ * @returns {string} - The mime type.
+ */
+const mimeTypeForFormat = (format) => {
+  switch (format) {
+    case "mov":
+      return "video/quicktime";
+    case "wav":
+      return "audio/wav";
+    case "mp3":
+      return "audio/mpeg";
+    case "mp4":
+      return "video/mp4";
+    case "mpeg":
+      return "video/mpeg";
+  }
+};

inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs

@@ -377,7 +377,11 @@ export const createSamplesDescriptor = (evalDescriptor, selectedScore) => {
     (previous, current) => {
       const text = inputString(current.input).join(" ");
       const scoreValue = evalDescriptor.score(current, selectedScore).value;
-      const scoreText = scoreValue ? String(scoreValue) : "";
+      const scoreText = scoreValue
+        ? String(scoreValue)
+        : current.error
+          ? String(current.error)
+          : "";
       previous[0] = Math.min(Math.max(previous[0], text.length), 300);
       previous[1] = Math.min(
         Math.max(previous[1], arrayToString(current.target).length),

inspect_ai/_view/www/src/types/log.d.ts

@@ -32,7 +32,6 @@ export type Limit = number | [unknown, unknown] | null;
 export type SampleId = string | number | (string | number)[] | null;
 export type Epochs = number | null;
 export type EpochsReducer = string[] | null;
-export type Trace = boolean | null;
 export type Name1 = string;
 export type Tools = string | string[];
 export type Approvers = ApproverPolicyConfig[];
@@ -112,35 +111,49 @@ export type Input =
       | ChatMessageAssistant
       | ChatMessageTool
     )[];
-export type Content = string | (ContentText | ContentImage)[];
+export type Content =
+  | string
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Type1 = "text";
 export type Text = string;
 export type Type2 = "image";
 export type Image = string;
 export type Detail = "auto" | "low" | "high";
+export type Type3 = "audio";
+export type Audio = string;
+export type Format = "wav" | "mp3";
+export type Type4 = "video";
+export type Video = string;
+export type Format1 = "mp4" | "mpeg" | "mov";
 export type Source = ("input" | "generate") | null;
 export type Role = "system";
-export type Content1 = string | (ContentText | ContentImage)[];
+export type Content1 =
+  | string
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source1 = ("input" | "generate") | null;
 export type Role1 = "user";
 export type ToolCallId = string | null;
-export type Content2 = string | (ContentText | ContentImage)[];
+export type Content2 =
+  | string
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source2 = ("input" | "generate") | null;
 export type Role2 = "assistant";
 export type ToolCalls = ToolCall[] | null;
 export type Id1 = string;
 export type Function = string;
-export type Type3 = "function";
+export type Type5 = "function";
 export type ParseError = string | null;
 export type Title = string | null;
-export type Format = "text" | "markdown";
+export type Format2 = "text" | "markdown";
 export type Content3 = string;
-export type Content4 = string | (ContentText | ContentImage)[];
+export type Content4 =
+  | string
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source3 = ("input" | "generate") | null;
 export type Role3 = "tool";
 export type ToolCallId1 = string | null;
 export type Function1 = string | null;
-export type Type4 =
+export type Type6 =
   | "parsing"
   | "timeout"
   | "unicode_decode"
@@ -218,7 +231,7 @@ export type JsonValue = unknown;
 export type Timestamp1 = string;
 export type Pending1 = boolean | null;
 export type Event1 = "sample_limit";
-export type Type5 = "message" | "time" | "token" | "operator";
+export type Type7 = "message" | "time" | "token" | "operator";
 export type Message2 = string;
 export type Limit1 = number | null;
 export type Timestamp2 = string;
@@ -244,8 +257,8 @@ export type Input2 = (
 )[];
 export type Name5 = string;
 export type Description = string;
-export type Type6 = "object";
-export type Type7 =
+export type Type8 = "object";
+export type Type9 =
   | ("string" | "integer" | "number" | "boolean" | "array" | "object" | "null")
   | null;
 export type Description1 = string | null;
@@ -265,7 +278,7 @@ export type Cache = ("read" | "write") | null;
 export type Timestamp5 = string;
 export type Pending5 = boolean | null;
 export type Event5 = "tool";
-export type Type8 = "function";
+export type Type10 = "function";
 export type Id3 = string;
 export type Function2 = string;
 export type Result =
@@ -274,7 +287,9 @@ export type Result =
   | boolean
   | ContentText
   | ContentImage
-  | (ContentText | ContentImage)[];
+  | ContentAudio
+  | ContentVideo
+  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Truncated = [unknown, unknown] | null;
 export type Timestamp6 = string;
 export type Pending6 = boolean | null;
@@ -324,13 +339,13 @@ export type Timestamp12 = string;
 export type Pending12 = boolean | null;
 export type Event12 = "step";
 export type Action = "begin" | "end";
-export type Type9 = string | null;
+export type Type11 = string | null;
 export type Name8 = string;
 export type Timestamp13 = string;
 export type Pending13 = boolean | null;
 export type Event13 = "subtask";
 export type Name9 = string;
-export type Type10 = string | null;
+export type Type12 = string | null;
 export type Events2 = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -379,7 +394,7 @@ export type Events = (
   | StepEvent
   | SubtaskEvent
 )[];
-export type Type11 = "context" | "time" | "message" | "token" | "operator";
+export type Type13 = "context" | "time" | "message" | "token" | "operator";
 export type Limit2 = number;
 export type Reductions = EvalSampleReductions[] | null;
 export type Scorer1 = string;
@@ -448,7 +463,6 @@ export interface EvalConfig {
   sample_id: SampleId;
   epochs: Epochs;
   epochs_reducer: EpochsReducer;
-  trace: Trace;
   approval: ApprovalPolicyConfig | null;
   fail_on_error: FailOnError;
   message_limit: MessageLimit;
@@ -614,6 +628,16 @@ export interface ContentImage {
   image: Image;
   detail: Detail;
 }
+export interface ContentAudio {
+  type: Type3;
+  audio: Audio;
+  format: Format;
+}
+export interface ContentVideo {
+  type: Type4;
+  video: Video;
+  format: Format1;
+}
 export interface ChatMessageUser {
   content: Content1;
   source: Source1;
@@ -630,7 +654,7 @@ export interface ToolCall {
   id: Id1;
   function: Function;
   arguments: Arguments;
-  type: Type3;
+  type: Type5;
   parse_error: ParseError;
   view: ToolCallContent | null;
 }
@@ -640,7 +664,7 @@ export interface Arguments {}
  */
 export interface ToolCallContent {
   title: Title;
-  format: Format;
+  format: Format2;
   content: Content3;
 }
 export interface ChatMessageTool {
@@ -652,7 +676,7 @@ export interface ChatMessageTool {
   error: ToolCallError | null;
 }
 export interface ToolCallError {
-  type: Type4;
+  type: Type6;
   message: Message1;
 }
 export interface ModelOutput {
@@ -735,7 +759,7 @@ export interface SampleLimitEvent {
   timestamp: Timestamp1;
   pending: Pending1;
   event: Event1;
-  type: Type5;
+  type: Type7;
   message: Message2;
   limit: Limit1;
 }
@@ -822,7 +846,7 @@ export interface ToolInfo {
  * Description of tool parameters object in JSON Schema format.
  */
 export interface ToolParams {
-  type: Type6;
+  type: Type8;
   properties: Properties;
   required: Required1;
   additionalProperties: Additionalproperties1;
@@ -834,7 +858,7 @@ export interface Properties {
  * Description of tool parameter in JSON Schema format.
  */
 export interface ToolParam {
-  type: Type7;
+  type: Type9;
   description: Description1;
   default: Default;
   enum: Enum;
@@ -897,7 +921,7 @@ export interface ToolEvent {
   timestamp: Timestamp5;
   pending: Pending5;
   event: Event5;
-  type: Type8;
+  type: Type10;
   id: Id3;
   function: Function2;
   arguments: Arguments1;
@@ -999,7 +1023,7 @@ export interface StepEvent {
   pending: Pending12;
   event: Event12;
   action: Action;
-  type: Type9;
+  type: Type11;
   name: Name8;
 }
 /**
@@ -1010,7 +1034,7 @@ export interface SubtaskEvent {
   pending: Pending13;
   event: Event13;
   name: Name9;
-  type: Type10;
+  type: Type12;
   input: Input4;
   result: Result1;
   events: Events2;
@@ -1026,7 +1050,7 @@ export interface Attachments {
   [k: string]: string;
 }
 export interface EvalSampleLimit {
-  type: Type11;
+  type: Type13;
   limit: Limit2;
 }
 export interface EvalSampleReductions {