inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/solver/_solver.py

@@ -29,36 +29,32 @@ from ._task_state import TaskState, set_sample_state
 
 @runtime_checkable
 class Generate(Protocol):
-    """Generate using the model and add the assistant message to the task state.
-
-    Args:
-       state (TaskState): Beginning task state.
-
-       tool_calls (Literal["loop", "single", "none"]): Resolve tool calls:
-          - `"loop"` resolves tools calls and then invokes `generate()`,
-             proceeding in a loop which terminates when there are no more
-             tool calls, or `message_limit` or `token_limit` is exceeded.
-             This is the default behavior.
-          - `"single"` resolves at most a single set of tool calls and then returns.
-          - `"none"` does not resolve tool calls at all (in this
-             case you will need to invoke `call_tools()` directly).
-
-       cache: (bool | CachePolicy):
-          Caching behaviour for generate responses (defaults to no caching).
-
-       **kwargs: Optional generation config arguments.
-
-    Returns:
-       Updated TaskState.
-    """
-
     async def __call__(
         self,
         state: TaskState,
         tool_calls: Literal["loop", "single", "none"] = "loop",
         cache: bool | CachePolicy = False,
         **kwargs: Unpack[GenerateConfigArgs],
-    ) -> TaskState: ...
+    ) -> TaskState:
+        """Generate using the model and add the assistant message to the task state.
+
+        Args:
+            state: Beginning task state.
+            tool_calls:
+                - `"loop"` resolves tools calls and then invokes `generate()`,
+                    proceeding in a loop which terminates when there are no more
+                    tool calls, or `message_limit` or `token_limit` is exceeded.
+                    This is the default behavior.
+                - `"single"` resolves at most a single set of tool calls and then returns.
+                - `"none"` does not resolve tool calls at all (in this
+                    case you will need to invoke `call_tools()` directly).
+            cache: Caching behaviour for generate responses (defaults to no caching).
+            **kwargs: Optional generation config arguments.
+
+        Returns:
+            Updated TaskState.
+        """
+        ...
 
 
 @dataclass(frozen=True)
@@ -74,28 +70,37 @@ class SolverSpec:
 
 @runtime_checkable
 class Solver(Protocol):
-    r"""Contribute to solving an evaluation task.
+    async def __call__(
+        self,
+        state: TaskState,
+        generate: Generate,
+    ) -> TaskState:
+        r"""Contribute to solving an evaluation task.
 
-    Contribute to the solution of a task by transforming a TaskState
-    (e.g. prompt enhancement, elicitation, etc.). Solvers return a
-    TaskState (which could simply be a modified version of the one
-    they were passed) and optionally may call the generate() function
-    to generate output (and a new TaskState with that output).
+        Transform a `TaskState`, returning the new state. Solvers may
+        optionally call the `generate()` function to create a new
+        state resulting from model generation. Solvers may also do
+        prompt engineering or other types of elicitation.
 
+        Args:
+          state: State for tasks being evaluated.
+          generate: Function for generating outputs.
 
-    Args:
-        state (TaskState): States for tasks being evaluated.
-        generate (Generate): Function for generating outputs.
+        Returns:
+          Updated TaskState.
 
-    Returns:
-        Updated TaskState.
-    """
+        Examples:
+          ```python
+          @solver
+          def prompt_cot(template: str) -> Solver:
+              def solve(state: TaskState, generate: Generate) -> TaskState:
+                  # insert chain of thought prompt
+                  return state
 
-    async def __call__(
-        self,
-        state: TaskState,
-        generate: Generate,
-    ) -> TaskState: ...
+              return solve
+          ```
+        """
+        ...
 
 
 P = ParamSpec("P")
@@ -144,7 +149,7 @@ def solver(
     r"""Decorator for registering solvers.
 
     Args:
-        name: (str | Callable[P, Solver]):
+        name:
             Optional name for solver. If the decorator has no name
             argument then the name of the underlying Callable[P, Solver]
             object will be used to automatically assign a name.
@@ -153,19 +158,15 @@ def solver(
         Solver with registry attributes.
 
     Examples:
-        @solver
-        def prompt_cot(state: TaskState, generate: Generate) -> None:
-            ...
-
-        @solver(name = "prompt_cot")
-        def cot(state: TaskState, generate: Generate) -> None:
-            ...
-
+        ```python
         @solver
         def prompt_cot(template: str) -> Solver:
-            def solve(state: TaskState, generate: Generate) -> None:
-                ...
+            def solve(state: TaskState, generate: Generate) -> TaskState:
+                # insert chain of thought prompt
+                return state
+
             return solve
+        ```
     """
 
     # create_solver_wrapper:

inspect_ai/solver/_task_state.py

@@ -31,17 +31,20 @@ class Choice:
     """
     A `Choice` represents a single choice in a multiple choice question.
 
-    It is only relevant for the `multiple_choice` solver and corresponding `choice` scorer.
+    It is only relevant for the `multiple_choice` solver and corresponding
+    `choice` scorer.
     """
 
     value: str
     """The original value of the choice from the `Sample`."""
 
     correct: bool | None
-    """Did the model think this choice satisfies the question? `None` indicates this has not been set yet"""
+    """Did the model think this choice satisfies the question? `None`
+    indicates this has not been set yet"""
 
     original_position: int
-    """Choices may be re-ordered during processing, this represents the original position in the sample's list of choices"""
+    """Choices may be re-ordered during processing, this represents the
+    original position in the sample's list of choices"""
 
 
 class Choices(Sequence[Choice]):
@@ -127,10 +130,10 @@ class TaskState:
     """
     The `TaskState` represents the internal state of the `Task` being run for a single `Sample`.
 
-    It's a mutable object that is updated by each solver during a sample's
-    evaluation. It allows us to maintain things like the message history between
-    the running `Task` and the model, the tools available to the model, the
-    final output of the model and whether or not it's completed yet.
+    The `TaskState` is passed to and returned from each solver during a sample's
+    evaluation. It allows us to manipulated the message history, the tools
+    available to the model, the final output of the model, and whether the task
+    is completed or has hit a limit.
     """
 
     def __init__(
@@ -149,73 +152,39 @@ class TaskState:
         metadata: dict[str, Any] = {},
     ) -> None:
         self._model = model
-        """Model name used for this task."""
-
-        self.sample_id = sample_id
-        """Unique id for sample."""
-
-        self.epoch = epoch
-        """Epoch number for sample."""
-
+        self._sample_id = sample_id
+        self._epoch = epoch
         self._input = input
-        """
-        The original input from the `Sample` for this `TaskState`.
-
-        Should be treated as immutable and not changed during the run, so that
-        it can be referenced or checked wherever needed. Access through `input`
-        or `input_text` only
-        """
-
-        self.target = target
-        """The scoring target for this `Sample`."""
-
-        self.metadata = metadata
-        """Metadata from the `Sample` for this `TaskState`"""
-
+        self._target = target
+        self._metadata = metadata
         self._messages: list[ChatMessage] = ChatMessageList(messages, self)
-        """
-        Chat conversation history for sample.
-
-        This will generally get appended to every time a `generate` call is made
-        to the model. Useful for both debug and for solvers/scorers to assess
-        model performance or choose the next step.
-        """
-
         self._tools: list[Tool] = []
-        """Tools available to the model."""
-
-        self.tool_choice: ToolChoice | None = None
-        """Tool choice directive."""
-
-        self.output = output if output else ModelOutput(model=str(model), choices=[])
-        """
-        The 'final' model output once we've completed all solving.
-
-        For simple evals this may just be the last `message` from the
-        conversation history, but more complex solvers may generate this in
-        different ways depending on what solvers are used..
-        """
-
+        self._output = output if output else ModelOutput(model=str(model))
         self._message_limit = message_limit
         self._token_limit = token_limit
         self._completed = completed
-
-        """Store for shared data"""
-        self.store = Store()
+        self._store = Store()
 
         if choices:
             self.choices = Choices(choices)
         else:
             self.choices = Choices([])
 
-        self.scores: dict[str, Score] | None = None
-        """Scores yielded by running task."""
-
     @property
     def model(self) -> ModelName:
         """Name of model being evaluated."""
         return self._model
 
+    @property
+    def sample_id(self) -> int | str:
+        """Unique id for sample."""
+        return self._sample_id
+
+    @property
+    def epoch(self) -> int:
+        """Epoch number for sample."""
+        return self._epoch
+
     @property
     def input(self) -> str | list[ChatMessage]:
         """Input from the `Sample`, should be considered immutable."""
@@ -253,9 +222,6 @@ class TaskState:
         engineering solvers). This property enables easy read and
         write access to the user chat prompt. Raises an
         exception if there is no user prompt
-
-        Returns:
-           First user `ChatMessage` in the task state.
         """
         prompt = next((m for m in self.messages if m.role == "user"), None)
         if prompt:
@@ -263,16 +229,63 @@ class TaskState:
         else:
             raise ValueError("user_prompt requested from TaskState but none available")
 
+    @property
+    def metadata(self) -> dict[str, Any]:
+        """Metadata from the `Sample` for this `TaskState`"""
+        return self._metadata
+
+    @metadata.setter
+    def metadata(self, metadata: dict[str, Any]) -> None:
+        self._metadata = metadata
+
     @property
     def messages(self) -> list[ChatMessage]:
-        """Messages in chat history"""
+        """
+        Chat conversation history for sample.
+
+        This will generally get appended to every time a `generate` call is made
+        to the model. Useful for both debug and for solvers/scorers to assess
+        model performance or choose the next step.
+        """
         return self._messages
 
     @messages.setter
     def messages(self, messages: list[ChatMessage]) -> None:
-        """Set messages in chat history."""
         self._messages = ChatMessageList(messages, self)
 
+    @property
+    def output(self) -> ModelOutput:
+        """
+        The 'final' model output once we've completed all solving.
+
+        For simple evals this may just be the last `message` from the
+        conversation history, but more complex solvers may set this directly.
+        """
+        return self._output
+
+    @output.setter
+    def output(self, output: ModelOutput) -> None:
+        self._output = output
+
+    @property
+    def store(self) -> Store:
+        """Store for shared data"""
+        return self._store
+
+    @property
+    def tools(self) -> list[Tool]:
+        """Tools available to the model."""
+        return self._tools
+
+    @tools.setter
+    def tools(self, tools: list[Tool | ToolDef]) -> None:
+        self._tools.clear()
+        for tool in tools:
+            self._tools.append(tool if isinstance(tool, Tool) else tool.as_tool())
+
+    tool_choice: ToolChoice | None = None
+    """Tool choice directive."""
+
     @property
     def max_messages(self) -> int | None:
         """Deprecated (use message_limit)."""
@@ -351,14 +364,12 @@ class TaskState:
         self._completed = completed
 
     @property
-    def tools(self) -> list[Tool]:
-        return self._tools
+    def target(self) -> Target:
+        """The scoring target for this `Sample`."""
+        return self._target
 
-    @tools.setter
-    def tools(self, tools: list[Tool | ToolDef]) -> None:
-        self._tools.clear()
-        for tool in tools:
-            self._tools.append(tool if isinstance(tool, Tool) else tool.as_tool())
+    scores: dict[str, Score] | None = None
+    """Scores yielded by running task."""
 
     def metadata_as(self, metadata_cls: Type[MT]) -> MT:
         """Pydantic model interface to metadata.

inspect_ai/solver/_use_tools.py

@@ -15,15 +15,15 @@ def use_tools(
     Inject tools into the task state to be used in generate().
 
     Args:
-        *tools (Tool | list[Tool]): One or more tools or lists of tools
-          to make available to the model. If no tools are passed, then
-          no change to the currently available set of `tools` is made.
-        tool_choice (ToolChoice | None): Directive indicating which
-          tools the model should use. If `None` is passed, then no
-          change to `tool_choice` is made.
-        append (bool): If `True`, then the passed-in tools are appended
-            to the existing tools; otherwise any existing tools are
-            replaced (the default)
+      *tools: One or more tools or lists of tools
+        to make available to the model. If no tools are passed, then
+        no change to the currently available set of `tools` is made.
+      tool_choice: Directive indicating which
+        tools the model should use. If `None` is passed, then no
+        change to `tool_choice` is made.
+      append: If `True`, then the passed-in tools are appended
+        to the existing tools; otherwise any existing tools are
+        replaced (the default)
 
     Returns:
         A solver that injects the tools and tool_choice into the task state.

inspect_ai/tool/__init__.py

@@ -19,14 +19,16 @@ from ._tool_call import (
 from ._tool_choice import ToolChoice, ToolFunction
 from ._tool_def import ToolDef
 from ._tool_info import ToolInfo
-from ._tool_params import ToolParam, ToolParams
+from ._tool_params import JSONType, ToolParam, ToolParams
 from ._tool_with import tool_with
+from ._tools._computer import computer
 from ._tools._execute import bash, python
 from ._tools._web_browser import web_browser
 from ._tools._web_search import web_search
 
 __all__ = [
     "bash",
+    "computer",
     "python",
     "web_browser",
     "web_search",
@@ -52,6 +54,7 @@ __all__ = [
     "ToolInfo",
     "ToolParam",
     "ToolParams",
+    "JSONType",
 ]
 
 _UTIL_MODULE_VERSION = "0.3.19"

inspect_ai/tool/_tool.py

@@ -40,10 +40,25 @@ ToolResult = (
     | ContentVideo
     | list[ContentText | ContentImage | ContentAudio | ContentVideo]
 )
+"""Valid types for results from tool calls."""
 
 
 class ToolError(Exception):
+    """Exception thrown from tool call.
+
+    If you throw a `ToolError` form within a tool call,
+    the error will be reported to the model for further
+    processing (rather than ending the sample). If you want
+    to raise a fatal error from a tool call use an appropriate
+    standard exception type (e.g. `RuntimeError`, `ValueError`, etc.)
+    """
+
     def __init__(self, message: str) -> None:
+        """Create a ToolError.
+
+        Args:
+          message: Error message to report to the model.
+        """
         super().__init__(message)
         self.message = message
 
@@ -68,11 +83,21 @@ class Tool(Protocol):
         r"""Additional tool that an agent can use to solve a task.
 
         Args:
-            *args (Any): Arguments for the tool.
-            **kwargs (Any): Keyword arguments for the tool.
+          *args: Arguments for the tool.
+          **kwargs: Keyword arguments for the tool.
 
         Returns:
             Result of tool call.
+
+        Examples:
+          ```python
+          @tool
+          def add() -> Tool:
+              async def execute(x: int, y: int) -> int:
+                  return x + y
+
+              return execute
+          ```
         """
         ...
 
@@ -130,25 +155,29 @@ def tool(
     r"""Decorator for registering tools.
 
     Args:
-        func (ToolType | None): Tool function
-        name (str | None):
-            Optional name for tool. If the decorator has no name
+        func: Tool function
+        name: Optional name for tool. If the decorator has no name
             argument then the name of the tool creation function
             will be used as the name of the tool.
-        viewer (ToolCallViewer | None): Provide a custom view
-            of tool call and context.
-        model_input (ToolCallModelInput | None): Provide a custom
-            function for playing back tool results as model input.
-        parallel (bool):
-            Does this tool support parallel execution?
-            (defaults to True).
-        prompt (str):
-            Deprecated (provide all descriptive information about
+        viewer: Provide a custom view of tool call and context.
+        model_input: Provide a custom function for playing back tool results as model input.
+        parallel: Does this tool support parallel execution? (defaults to `True`).
+        prompt: Deprecated (provide all descriptive information about
             the tool within the tool function's doc comment)
 
 
     Returns:
         Tool with registry attributes.
+
+    Examples:
+        ```python
+        @tool
+        def add() -> Tool:
+            async def execute(x: int, y: int) -> int:
+                return x + y
+
+            return execute
+        ```
     """
     if prompt:
         from inspect_ai._util.logger import warn_once

inspect_ai/tool/_tool_call.py

@@ -13,10 +13,10 @@ class ToolCallContent(BaseModel):
     """Optional (plain text) title for tool call content."""
 
     format: Literal["text", "markdown"]
-    """Format."""
+    """Format (text or markdown)."""
 
     content: str
-    """Content."""
+    """Text or markdown content."""
 
 
 class ToolCallView(BaseModel):
@@ -56,6 +56,8 @@ class ToolCall:
 
 @dataclass
 class ToolCallError:
+    """Error raised by a tool call."""
+
     type: Literal[
         "parsing",
         "timeout",
@@ -67,8 +69,10 @@ class ToolCallError:
         "approval",
         "unknown",
     ]
+    """Error type."""
 
     message: str
+    """Error message."""
 
 
 ToolCallViewer = Callable[[ToolCall], ToolCallView]

inspect_ai/tool/_tool_choice.py

@@ -4,8 +4,10 @@ from typing import Literal, Union
 
 @dataclass
 class ToolFunction:
+    """Indicate that a specific tool function should be called."""
+
     name: str
-    """The name of the function to call."""
+    """The name of the tool function to call."""
 
 
 ToolChoice = Union[Literal["auto", "any", "none"], ToolFunction]

inspect_ai/tool/_tool_def.py

@@ -25,6 +25,8 @@ from ._tool_params import ToolParams
 
 
 class ToolDef:
+    """Tool definition."""
+
     def __init__(
         self,
         tool: Callable[..., Any],
@@ -35,19 +37,19 @@ class ToolDef:
         viewer: ToolCallViewer | None = None,
         model_input: ToolCallModelInput | None = None,
     ) -> None:
-        """Tool definition.
+        """Create a tool definition.
 
         Args:
-          tool (Callable[..., Any]): Callable to execute tool.
-          name (str | None): Name of tool. Discovered automatically if not specified.
-          description (str | None): Description of tool. Discovered automatically
+          tool: Callable to execute tool.
+          name: Name of tool. Discovered automatically if not specified.
+          description: Description of tool. Discovered automatically
             by parsing doc comments if not specified.
-          parameters (dict[str,str] | ToolParams | None): Tool parameter descriptions and types.
+          parameters: Tool parameter descriptions and types.
              Discovered automatically by parsing doc comments if not specified.
-          parallel (bool | None): Does the tool support parallel execution
+          parallel: Does the tool support parallel execution
              (defaults to True if not specified)
-          viewer (ToolCallViewer | None): Optional tool call viewer implementation.
-          model_input (ToolCallModelInput | None): Optional function that determines how
+          viewer: Optional tool call viewer implementation.
+          model_input: Optional function that determines how
               tool call results are played back as model input.
 
         Returns:

inspect_ai/tool/_tool_params.py

@@ -14,20 +14,44 @@ class ToolParam(BaseModel):
     """Description of tool parameter in JSON Schema format."""
 
     type: JSONType | None = Field(default=None)
+    """JSON type of tool parameter."""
+
     description: str | None = Field(default=None)
+    """Parameter description."""
+
     default: Any = Field(default=None)
+    """Default value for parameter."""
+
     enum: list[Any] | None = Field(default=None)
+    """Valid values for enum parameters."""
+
     items: Optional["ToolParam"] = Field(default=None)
+    """Valid type for array parameters."""
+
     properties: dict[str, "ToolParam"] | None = Field(default=None)
+    """Valid fields for object parametrs."""
+
     additionalProperties: Optional["ToolParam"] | bool | None = Field(default=None)
+    """Are additional properties allowed?"""
+
     anyOf: list["ToolParam"] | None = Field(default=None)
+    """Valid types for union parameters."""
+
     required: list[str] | None = Field(default=None)
+    """Required fields for object parameters."""
 
 
 class ToolParams(BaseModel):
     """Description of tool parameters object in JSON Schema format."""
 
     type: Literal["object"] = Field(default="object")
+    """Params type (always 'object')"""
+
     properties: dict[str, ToolParam] = Field(default_factory=dict)
+    """Tool function parameters."""
+
     required: list[str] = Field(default_factory=list)
+    """List of required fields."""
+
     additionalProperties: bool = Field(default=False)
+    """Are additional object properties allowed? (always `False`)"""

inspect_ai/tool/_tool_with.py

@@ -25,14 +25,14 @@ def tool_with(
     """Tool with modifications to name and descriptions.
 
     Args:
-       tool (Tool): Tool instance to copy and add descriptions to.
-       name (str | None): Tool name (optional).
-       description (str | None): Tool description (optional).
-       parameters (dict[str,str] | None): Parameter descriptions (optional)
-       parallel (bool | None): Does the tool support parallel execution
+       tool: Tool instance to copy and add descriptions to.
+       name: Tool name (optional).
+       description: Tool description (optional).
+       parameters: Parameter descriptions (optional)
+       parallel: Does the tool support parallel execution
           (defaults to True if not specified)
-       viewer (ToolCallViewer | None): Optional tool call viewer implementation.
-       model_input (ToolCallModelInput | None): Optional function that determines how
+       viewer: Optional tool call viewer implementation.
+       model_input: Optional function that determines how
            tool call results are played back as model input.
 
     Returns: