inspect-ai 0.3.72__py3-none-any.whl → 0.3.73__py3-none-any.whl

inspect_ai/_view/www/src/types/log.d.ts

@@ -21,11 +21,7 @@ export type Location = string | null;
 export type Samples = number | null;
 export type SampleIds = (number | string)[] | null;
 export type Shuffled = boolean | null;
-/**
- * @minItems 1
- * @maxItems 2
- */
-export type SandboxEnvironmentSpec = [unknown] | [unknown, unknown];
+export type Type = string;
 export type Model = string;
 export type ModelBaseUrl = string | null;
 export type Limit = number | [unknown, unknown] | null;
@@ -49,7 +45,7 @@ export type LogSamples = boolean | null;
 export type LogImages = boolean | null;
 export type LogBuffer = number | null;
 export type ScoreDisplay = boolean | null;
-export type Type = "git";
+export type Type1 = "git";
 export type Origin = string;
 export type Commit = string;
 export type Metadata = {} | null;
@@ -105,12 +101,26 @@ export type CachePrompt = "auto" | boolean | null;
 export type ReasoningEffort = ("low" | "medium" | "high") | null;
 export type ReasoningTokens = number | null;
 export type ReasoningHistory = ("none" | "all" | "last" | "auto") | null;
+export type Name5 = string;
+export type Type2 =
+  | ("string" | "integer" | "number" | "boolean" | "array" | "object" | "null")
+  | null;
+export type Description = string | null;
+export type Enum = unknown[] | null;
+export type Properties = {
+  [k: string]: JSONSchema;
+} | null;
+export type Additionalproperties = JSONSchema | boolean | null;
+export type Anyof = JSONSchema[] | null;
+export type Required = string[] | null;
+export type Description1 = string | null;
+export type Strict = boolean | null;
 export type TotalSamples = number;
 export type CompletedSamples = number;
-export type Name5 = string;
+export type Name6 = string;
 export type Scorer = string;
 export type Reducer = string | null;
-export type Name6 = string;
+export type Name7 = string;
 export type Value = number;
 export type Metadata2 = {} | null;
 export type Metadata3 = {} | null;
@@ -138,7 +148,7 @@ export type Input =
       | ChatMessageAssistant
       | ChatMessageTool
     )[];
-export type Role = "system";
+export type Id1 = string;
 export type Content =
   | string
   | (
@@ -148,23 +158,24 @@ export type Content =
       | ContentAudio
       | ContentVideo
     )[];
-export type Type1 = "text";
+export type Type3 = "text";
 export type Text = string;
-export type Type2 = "reasoning";
+export type Type4 = "reasoning";
 export type Reasoning = string;
 export type Signature = string | null;
 export type Redacted = boolean;
-export type Type3 = "image";
+export type Type5 = "image";
 export type Image = string;
 export type Detail = "auto" | "low" | "high";
-export type Type4 = "audio";
+export type Type6 = "audio";
 export type Audio = string;
 export type Format = "wav" | "mp3";
-export type Type5 = "video";
+export type Type7 = "video";
 export type Video = string;
 export type Format1 = "mp4" | "mpeg" | "mov";
 export type Source = ("input" | "generate") | null;
-export type Role1 = "user";
+export type Role = "system";
+export type Id2 = string;
 export type Content1 =
   | string
   | (
@@ -175,8 +186,9 @@ export type Content1 =
       | ContentVideo
     )[];
 export type Source1 = ("input" | "generate") | null;
+export type Role1 = "user";
 export type ToolCallId = string[] | null;
-export type Role2 = "assistant";
+export type Id3 = string;
 export type Content2 =
   | string
   | (
@@ -187,15 +199,16 @@ export type Content2 =
       | ContentVideo
     )[];
 export type Source2 = ("input" | "generate") | null;
+export type Role2 = "assistant";
 export type ToolCalls = ToolCall[] | null;
-export type Id1 = string;
+export type Id4 = string;
 export type Function = string;
-export type Type6 = "function";
+export type Type8 = "function";
 export type ParseError = string | null;
 export type Title = string | null;
 export type Format2 = "text" | "markdown";
 export type Content3 = string;
-export type Role3 = "tool";
+export type Id5 = string;
 export type Content4 =
   | string
   | (
@@ -206,9 +219,10 @@ export type Content4 =
       | ContentVideo
     )[];
 export type Source3 = ("input" | "generate") | null;
+export type Role3 = "tool";
 export type ToolCallId1 = string | null;
 export type Function1 = string | null;
-export type Type7 =
+export type Type9 =
   | "parsing"
   | "timeout"
   | "unicode_decode"
@@ -277,7 +291,7 @@ export type Input1 =
     )[];
 export type Choices2 = string[] | null;
 export type Target1 = string | string[];
-export type Id2 = number | string | null;
+export type Id6 = number | string | null;
 export type Metadata8 = {} | null;
 export type Files1 = {
   [k: string]: string;
@@ -288,7 +302,7 @@ export type Timestamp1 = string;
 export type WorkingStart1 = number;
 export type Pending1 = boolean | null;
 export type Event1 = "sample_limit";
-export type Type8 =
+export type Type10 =
   | "message"
   | "time"
   | "working"
@@ -335,25 +349,14 @@ export type Input3 = (
   | ChatMessageAssistant
   | ChatMessageTool
 )[];
-export type Name7 = string;
-export type Description = string;
-export type Type9 = "object";
-export type Type10 =
-  | ("string" | "integer" | "number" | "boolean" | "array" | "object" | "null")
-  | null;
-export type Description1 = string | null;
-export type Enum = unknown[] | null;
-export type Properties1 = {
-  [k: string]: ToolParam;
-} | null;
-export type Additionalproperties = ToolParam | boolean | null;
-export type Anyof = ToolParam[] | null;
-export type Required = string[] | null;
+export type Name8 = string;
+export type Description2 = string;
+export type Type11 = "object";
 export type Required1 = string[];
 export type Additionalproperties1 = boolean;
 export type Tools1 = ToolInfo[];
 export type ToolChoice = ("auto" | "any" | "none") | ToolFunction;
-export type Name8 = string;
+export type Name9 = string;
 export type Error1 = string | null;
 export type Cache = ("read" | "write") | null;
 export type Time1 = number | null;
@@ -363,8 +366,8 @@ export type Timestamp6 = string;
 export type WorkingStart6 = number;
 export type Pending6 = boolean | null;
 export type Event6 = "tool";
-export type Type11 = "function";
-export type Id3 = string;
+export type Type12 = "function";
+export type Id7 = string;
 export type Function2 = string;
 export type Result1 =
   | string
@@ -416,7 +419,7 @@ export type Timestamp11 = string;
 export type WorkingStart11 = number;
 export type Pending11 = boolean | null;
 export type Event11 = "logger";
-export type Name9 = string | null;
+export type Name10 = string | null;
 export type Level =
   | "debug"
   | "trace"
@@ -441,14 +444,14 @@ export type WorkingStart13 = number;
 export type Pending13 = boolean | null;
 export type Event13 = "step";
 export type Action1 = "begin" | "end";
-export type Type12 = string | null;
-export type Name10 = string;
+export type Type13 = string | null;
+export type Name11 = string;
 export type Timestamp14 = string;
 export type WorkingStart14 = number;
 export type Pending14 = boolean | null;
 export type Event14 = "subtask";
-export type Name11 = string;
-export type Type13 = string | null;
+export type Name12 = string;
+export type Type14 = string | null;
 export type Events2 = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -507,7 +510,7 @@ export type Events = (
 export type TotalTime = number | null;
 export type WorkingTime3 = number | null;
 export type Uuid = string | null;
-export type Type14 =
+export type Type15 =
   | "context"
   | "time"
   | "working"
@@ -588,6 +591,16 @@ export interface EvalDataset {
   sample_ids: SampleIds;
   shuffled: Shuffled;
 }
+/**
+ * Specification of a SandboxEnvironment.
+ */
+export interface SandboxEnvironmentSpec {
+  type: Type;
+  config: Config;
+}
+export interface Config {
+  [k: string]: unknown;
+}
 export interface ModelArgs {}
 /**
  * Configuration used for evaluation.
@@ -642,7 +655,7 @@ export interface Params {}
  * Git revision for evaluation.
  */
 export interface EvalRevision {
-  type: Type;
+  type: Type1;
   origin: Origin;
   commit: Commit;
 }
@@ -704,6 +717,33 @@ export interface GenerateConfig {
   reasoning_effort: ReasoningEffort;
   reasoning_tokens: ReasoningTokens;
   reasoning_history: ReasoningHistory;
+  response_schema: ResponseSchema | null;
+}
+/**
+ * Schema for model response when using Structured Output.
+ */
+export interface ResponseSchema {
+  name: Name5;
+  json_schema: JSONSchema;
+  description: Description1;
+  strict: Strict;
+}
+/**
+ * JSON Schema for type.
+ */
+export interface JSONSchema {
+  type: Type2;
+  description: Description;
+  default: Default;
+  enum: Enum;
+  items: JSONSchema | null;
+  properties: Properties;
+  additionalProperties: Additionalproperties;
+  anyOf: Anyof;
+  required: Required;
+}
+export interface Default {
+  [k: string]: unknown;
 }
 /**
  * Scoring results from evaluation.
@@ -718,7 +758,7 @@ export interface EvalResults {
  * Score for evaluation task.
  */
 export interface EvalScore {
-  name: Name5;
+  name: Name6;
   scorer: Scorer;
   reducer: Reducer;
   params: Params2;
@@ -733,7 +773,7 @@ export interface Metrics2 {
  * Metric for evaluation score.
  */
 export interface EvalMetric {
-  name: Name6;
+  name: Name7;
   value: Value;
   params: Params3;
   metadata: Metadata2;
@@ -799,15 +839,16 @@ export interface EvalSample {
  * System chat message.
  */
 export interface ChatMessageSystem {
-  role: Role;
+  id: Id1;
   content: Content;
   source: Source;
+  role: Role;
 }
 /**
  * Text content.
  */
 export interface ContentText {
-  type: Type1;
+  type: Type3;
   text: Text;
 }
 /**
@@ -816,7 +857,7 @@ export interface ContentText {
  * See the specification for [thinking blocks](https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#understanding-thinking-blocks) for Claude models.
  */
 export interface ContentReasoning {
-  type: Type2;
+  type: Type4;
   reasoning: Reasoning;
   signature: Signature;
   redacted: Redacted;
@@ -825,7 +866,7 @@ export interface ContentReasoning {
  * Image content.
  */
 export interface ContentImage {
-  type: Type3;
+  type: Type5;
   image: Image;
   detail: Detail;
 }
@@ -833,7 +874,7 @@ export interface ContentImage {
  * Audio content.
  */
 export interface ContentAudio {
-  type: Type4;
+  type: Type6;
   audio: Audio;
   format: Format;
 }
@@ -841,7 +882,7 @@ export interface ContentAudio {
  * Video content.
  */
 export interface ContentVideo {
-  type: Type5;
+  type: Type7;
   video: Video;
   format: Format1;
 }
@@ -849,25 +890,27 @@ export interface ContentVideo {
  * User chat message.
  */
 export interface ChatMessageUser {
-  role: Role1;
+  id: Id2;
   content: Content1;
   source: Source1;
+  role: Role1;
   tool_call_id: ToolCallId;
 }
 /**
  * Assistant chat message.
  */
 export interface ChatMessageAssistant {
-  role: Role2;
+  id: Id3;
   content: Content2;
   source: Source2;
+  role: Role2;
   tool_calls: ToolCalls;
 }
 export interface ToolCall {
-  id: Id1;
+  id: Id4;
   function: Function;
   arguments: Arguments;
-  type: Type6;
+  type: Type8;
   parse_error: ParseError;
   view: ToolCallContent | null;
 }
@@ -884,15 +927,16 @@ export interface ToolCallContent {
  * Tool chat message.
  */
 export interface ChatMessageTool {
-  role: Role3;
+  id: Id5;
   content: Content4;
   source: Source3;
+  role: Role3;
   tool_call_id: ToolCallId1;
   function: Function1;
   error: ToolCallError | null;
 }
 export interface ToolCallError {
-  type: Type7;
+  type: Type9;
   message: Message1;
 }
 /**
@@ -966,7 +1010,7 @@ export interface Sample {
   input: Input1;
   choices: Choices2;
   target: Target1;
-  id: Id2;
+  id: Id6;
   metadata: Metadata8;
   sandbox: SandboxEnvironmentSpec | null;
   files: Files1;
@@ -980,7 +1024,7 @@ export interface SampleLimitEvent {
   working_start: WorkingStart1;
   pending: Pending1;
   event: Event1;
-  type: Type8;
+  type: Type10;
   message: Message2;
   limit: Limit1;
 }
@@ -1082,41 +1126,24 @@ export interface ModelEvent {
  * ```
  */
 export interface ToolInfo {
-  name: Name7;
-  description: Description;
+  name: Name8;
+  description: Description2;
   parameters: ToolParams;
 }
 /**
  * Description of tool parameters object in JSON Schema format.
  */
 export interface ToolParams {
-  type: Type9;
-  properties: Properties;
+  type: Type11;
+  properties: Properties1;
   required: Required1;
   additionalProperties: Additionalproperties1;
 }
-export interface Properties {
-  [k: string]: ToolParam;
-}
-/**
- * Description of tool parameter in JSON Schema format.
- */
-export interface ToolParam {
-  type: Type10;
-  description: Description1;
-  default: Default;
-  enum: Enum;
-  items: ToolParam | null;
-  properties: Properties1;
-  additionalProperties: Additionalproperties;
-  anyOf: Anyof;
-  required: Required;
-}
-export interface Default {
-  [k: string]: unknown;
+export interface Properties1 {
+  [k: string]: JSONSchema;
 }
 export interface ToolFunction {
-  name: Name8;
+  name: Name9;
 }
 /**
  * Model generation options.
@@ -1146,6 +1173,7 @@ export interface GenerateConfig1 {
   reasoning_effort: ReasoningEffort;
   reasoning_tokens: ReasoningTokens;
   reasoning_history: ReasoningHistory;
+  response_schema: ResponseSchema | null;
 }
 /**
  * Model call (raw request/response data).
@@ -1169,8 +1197,8 @@ export interface ToolEvent {
   working_start: WorkingStart6;
   pending: Pending6;
   event: Event6;
-  type: Type11;
-  id: Id3;
+  type: Type12;
+  id: Id7;
   function: Function2;
   arguments: Arguments1;
   view: ToolCallContent | null;
@@ -1260,7 +1288,7 @@ export interface LoggerEvent {
  * Message written to Python log.
  */
 export interface LoggingMessage {
-  name: Name9;
+  name: Name10;
   level: Level;
   message: Message4;
   created: Created1;
@@ -1288,8 +1316,8 @@ export interface StepEvent {
   pending: Pending13;
   event: Event13;
   action: Action1;
-  type: Type12;
-  name: Name10;
+  type: Type13;
+  name: Name11;
 }
 /**
  * Subtask spawned.
@@ -1299,8 +1327,8 @@ export interface SubtaskEvent {
   working_start: WorkingStart14;
   pending: Pending14;
   event: Event14;
-  name: Name11;
-  type: Type13;
+  name: Name12;
+  type: Type14;
   input: Input5;
   result: Result2;
   events: Events2;
@@ -1321,7 +1349,7 @@ export interface Attachments {
  * Limit encontered by sample.
  */
 export interface EvalSampleLimit {
-  type: Type14;
+  type: Type15;
   limit: Limit2;
 }
 /**

inspect_ai/approval/_human/manager.py

@@ -1,9 +1,8 @@
-import asyncio
 import uuid
-from asyncio import Future
 from contextvars import ContextVar
-from typing import Callable, Literal, NamedTuple, cast
+from typing import Callable, Literal, NamedTuple
 
+from inspect_ai._util.future import Future
 from inspect_ai.solver._task_state import TaskState
 from inspect_ai.tool._tool_call import ToolCall, ToolCallView
 
@@ -37,7 +36,6 @@ class HumanApprovalManager:
         from inspect_ai.log._samples import sample_active
 
         id = str(uuid.uuid4())
-        future = cast(Future[Approval], asyncio.get_event_loop().create_future())
         sample = sample_active()
         assert sample
         assert sample.sample.id is not None
@@ -48,7 +46,7 @@ class HumanApprovalManager:
             id=sample.sample.id,
             epoch=sample.epoch,
         )
-        self._approval_requests[id] = (pending, future)
+        self._approval_requests[id] = (pending, Future[Approval]())
         self._notify_change("add")
         return id
 
@@ -58,7 +56,7 @@ class HumanApprovalManager:
 
     async def wait_for_approval(self, id: str) -> Approval:
         _, future = self._approval_requests[id]
-        return await future
+        return await future.result()
 
     def on_change(
         self, callback: Callable[[Literal["add", "remove"]], None]
@@ -77,16 +75,14 @@ class HumanApprovalManager:
     def complete_approval(self, id: str, result: Approval) -> None:
         if id in self._approval_requests:
             _, future = self._approval_requests[id]
-            if not future.done():
-                future.set_result(result)
+            future.set_result(result)
             del self._approval_requests[id]
             self._notify_change("remove")
 
     def fail_approval(self, id: str, error: Exception) -> None:
         if id in self._approval_requests:
             _, future = self._approval_requests[id]
-            if not future.done():
-                future.set_exception(error)
+            future.set_exception(error)
             del self._approval_requests[id]
             self._notify_change("remove")
 

inspect_ai/approval/_human/panel.py

@@ -1,6 +1,6 @@
-from asyncio import CancelledError
 from typing import Callable, Literal
 
+import anyio
 from rich.console import RenderableType
 from rich.text import Text
 from textual.app import ComposeResult
@@ -44,7 +44,7 @@ async def panel_approval(
     )
     try:
         return await approvals.wait_for_approval(id)
-    except CancelledError:
+    except anyio.get_cancelled_exc_class():
         approvals.withdraw_request(id)
         raise
 

inspect_ai/dataset/_sources/util.py

@@ -63,12 +63,13 @@ def message_with_resolved_content(
     message: ChatMessage, resolver: Callable[[str], str]
 ) -> ChatMessage:
     if isinstance(message, ChatMessageUser) and not isinstance(message.content, str):
-        return ChatMessageUser(
-            content=[
-                chat_content_with_resolved_content(content, resolver)
-                for content in message.content
-            ],
-            source=message.source,
+        return message.model_copy(
+            update=dict(
+                content=[
+                    chat_content_with_resolved_content(content, resolver)
+                    for content in message.content
+                ],
+            )
         )
     else:
         return message

inspect_ai/log/__init__.py

@@ -7,9 +7,11 @@ from ._file import (
     EvalLogInfo,
     list_eval_logs,
     read_eval_log,
+    read_eval_log_async,
     read_eval_log_sample,
     read_eval_log_samples,
     write_eval_log,
+    write_eval_log_async,
     write_log_dir_manifest,
 )
 from ._log import (
@@ -93,11 +95,13 @@ __all__ = [
     "convert_eval_logs",
     "list_eval_logs",
     "read_eval_log",
+    "read_eval_log_async",
     "read_eval_log_sample",
     "read_eval_log_samples",
     "condense_sample",
     "resolve_sample_attachments",
     "write_eval_log",
+    "write_eval_log_async",
     "write_log_dir_manifest",
     "retryable_eval_logs",
     "bundle_log_dir",