inspect-ai 0.3.61__py3-none-any.whl → 0.3.62__py3-none-any.whl

inspect_ai/_cli/eval.py

@@ -385,6 +385,14 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         help="Constrains effort on reasoning for reasoning models. Open AI o1 models only.",
         envvar="INSPECT_EVAL_REASONING_EFFORT",
     )
+    @click.option(
+        "--reasoning-history/--no-reasoning-history",
+        type=bool,
+        is_flag=True,
+        default=True,
+        help="Include reasoning in chat message history sent to generate.",
+        envvar="INSPECT_EVAL_REASONING_HISTORY",
+    )
     @click.option(
         "--log-format",
         type=click.Choice(["eval", "json"], case_sensitive=False),
@@ -444,6 +452,7 @@ def eval_command(
     max_tool_output: int | None,
     cache_prompt: str | None,
     reasoning_effort: str | None,
+    reasoning_history: bool | None,
     message_limit: int | None,
     token_limit: int | None,
     time_limit: int | None,
@@ -603,6 +612,7 @@ def eval_set_command(
     max_tool_output: int | None,
     cache_prompt: str | None,
     reasoning_effort: str | None,
+    reasoning_history: bool | None,
     message_limit: int | None,
     token_limit: int | None,
     time_limit: int | None,
@@ -841,6 +851,9 @@ def config_from_locals(locals: dict[str, Any]) -> GenerateConfigArgs:
             if key == "internal_tools":
                 if value is not False:
                     value = None
+            if key == "reasoning_history":
+                if value is not False:
+                    value = None
             config[key] = value  # type: ignore
     return config
 

inspect_ai/_cli/view.py

@@ -63,6 +63,10 @@ def start(
     INSPECT_VIEW_AUTHORIZATION_TOKEN = "INSPECT_VIEW_AUTHORIZATION_TOKEN"
     authorization = os.environ.get(INSPECT_VIEW_AUTHORIZATION_TOKEN, None)
     if authorization:
+        # this indicates we are in vscode -- we want to set the log level to HTTP
+        # in vscode, updated versions of the extension do this but we set it
+        # manually here as a temporary bridge for running against older versions
+        common["log_level"] = "HTTP"
         del os.environ[INSPECT_VIEW_AUTHORIZATION_TOKEN]
         os.unsetenv(INSPECT_VIEW_AUTHORIZATION_TOKEN)
 

inspect_ai/_display/textual/widgets/transcript.py

@@ -15,6 +15,7 @@ from inspect_ai._util.transcript import (
     set_transcript_markdown_options,
     transcript_function,
     transcript_markdown,
+    transcript_reasoning,
     transcript_separator,
 )
 from inspect_ai.log._samples import ActiveSample
@@ -33,7 +34,11 @@ from inspect_ai.log._transcript import (
     SubtaskEvent,
     ToolEvent,
 )
-from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+from inspect_ai.model._chat_message import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ChatMessageUser,
+)
 from inspect_ai.model._render import messages_preceding_assistant
 from inspect_ai.tool._tool import ToolResult
 from inspect_ai.tool._tool_transcript import transcript_tool_call
@@ -171,8 +176,8 @@ def render_model_event(event: ModelEvent) -> EventDisplay:
     # content
     content: list[RenderableType] = []
 
-    def append_message(message: ChatMessage, text: str | None = None) -> None:
-        content.extend(render_message(message, text))
+    def append_message(message: ChatMessage) -> None:
+        content.extend(render_message(message))
 
     # render preceding messages
     preceding = messages_preceding_assistant(event.input)
@@ -309,16 +314,17 @@ def render_as_json(json: Any) -> RenderableType:
     )
 
 
-def render_message(
-    message: ChatMessage, text: str | None = None
-) -> list[RenderableType]:
+def render_message(message: ChatMessage) -> list[RenderableType]:
     content: list[RenderableType] = [
         Text(message.role.capitalize(), style="bold"),
         Text(),
     ]
-    text = text or message.text
-    if text:
-        content.extend([transcript_markdown(text.strip(), escape=True)])
+
+    if isinstance(message, ChatMessageAssistant) and message.reasoning:
+        content.extend(transcript_reasoning(message.reasoning))
+
+    if message.text:
+        content.extend([transcript_markdown(message.text.strip(), escape=True)])
     return content
 
 

inspect_ai/_eval/task/error.py

@@ -8,28 +8,24 @@ class SampleErrorHandler:
         self.fail_on_error = True if fail_on_error is None else fail_on_error
         self.total_samples = float(total_samples)
 
-    def __call__(self, ex: BaseException) -> EvalError:
+    def __call__(self, ex: BaseException) -> tuple[EvalError, BaseException | None]:
         # increment error count
         self.error_count += 1
 
         # create error (we may return it)
-        def sample_error() -> EvalError:
-            return eval_error(ex, type(ex), ex, ex.__traceback__)
+        def sample_error(
+            *, raise_error: bool
+        ) -> tuple[EvalError, BaseException | None]:
+            return eval_error(
+                ex, type(ex), ex, ex.__traceback__
+            ), ex if raise_error else None
 
         # check against limits
         if isinstance(self.fail_on_error, bool):
-            if self.fail_on_error:
-                raise ex
-            else:
-                return sample_error()
+            return sample_error(raise_error=self.fail_on_error)
         else:
             if self.fail_on_error < 1:
                 max_errors = self.fail_on_error * self.total_samples
-                if self.error_count >= max_errors:
-                    raise ex
-                else:
-                    return sample_error()
-            elif self.error_count >= self.fail_on_error:
-                raise ex
+                return sample_error(raise_error=self.error_count >= max_errors)
             else:
-                return sample_error()
+                return sample_error(raise_error=self.error_count >= self.fail_on_error)

inspect_ai/_eval/task/run.py

@@ -496,7 +496,7 @@ async def task_run_sample(
     logger: TaskLogger | None,
     log_images: bool,
     sample_source: EvalSampleSource | None,
-    sample_error: Callable[[BaseException], EvalError],
+    sample_error: SampleErrorHandler,
     sample_complete: Callable[[dict[str, SampleScore]], None],
     fails_on_error: bool,
     time_limit: int | None,
@@ -548,12 +548,12 @@ async def task_run_sample(
     )
 
     # helper to handle exceptions (will throw if we've exceeded the limit)
-    def handle_error(ex: BaseException) -> EvalError:
+    def handle_error(ex: BaseException) -> tuple[EvalError, BaseException | None]:
         err = sample_error(ex)
         py_logger.warning(
             f"Sample error (id: {sample.id}, epoch: {state.epoch}): {exception_message(ex)})"
         )
-        transcript()._event(ErrorEvent(error=err))
+        transcript()._event(ErrorEvent(error=err[0]))
         return err
 
     # solver loop
@@ -572,6 +572,7 @@ async def task_run_sample(
         ) as active,
     ):
         error: EvalError | None = None
+        raise_error: BaseException | None = None
         results: dict[str, SampleScore] = {}
         try:
             async with sandboxenv_cm:
@@ -640,7 +641,7 @@ async def task_run_sample(
                                 state = sample_state() or state
                             case "error":
                                 # default error handling
-                                error = handle_error(ex)
+                                error, raise_error = handle_error(ex)
 
                     else:
                         raise
@@ -660,7 +661,7 @@ async def task_run_sample(
                     state.completed = True
 
                 except BaseException as ex:
-                    error = handle_error(ex)
+                    error, raise_error = handle_error(ex)
 
                 # set timeout for scoring. if the original timeout was hit we still
                 # want to provide opportunity for scoring, but we don't necessarily
@@ -737,11 +738,10 @@ async def task_run_sample(
                         )
 
                     # handle error (this will throw if we've exceeded the limit)
-                    error = handle_error(ex)
+                    error, raise_error = handle_error(ex)
 
-        # handle sandboxenv init errors
         except Exception as ex:
-            error = handle_error(ex)
+            error, raise_error = handle_error(ex)
 
         # complete the sample
         progress(SAMPLE_TOTAL_PROGRESS_UNITS)
@@ -772,6 +772,8 @@ async def task_run_sample(
             if results is not None:
                 sample_complete(results)
             return results
+        elif raise_error:
+            raise raise_error
         else:
             return None
 

inspect_ai/_util/transcript.py

@@ -111,6 +111,17 @@ def transcript_panel(
     )
 
 
+def transcript_reasoning(reasoning: str) -> list[RenderableType]:
+    content: list[RenderableType] = []
+    content.append(
+        transcript_markdown(
+            f"**<think>**  \n{reasoning}  \n**</think>**\n\n", escape=True
+        )
+    )
+    content.append(Text())
+    return content
+
+
 def transcript_separator(title: str, color: str) -> RenderableType:
     return Rule(title=title, style=f"{color} bold", align="center", end="\n\n")
 

inspect_ai/_view/www/dist/assets/index.css

@@ -15735,6 +15735,7 @@ pre.ap-terminal.ap-cursor-on .ap-line .ap-cursor.ap-inverse {
 }
 pre.ap-terminal:not(.ap-blink) .ap-line .ap-blink {
   color: transparent;
+  border-color: transparent;
 }
 pre.ap-terminal .ap-bright {
   font-weight: bold;