inspect-ai 0.3.61__py3-none-any.whl → 0.3.63__py3-none-any.whl

inspect_ai/_cli/eval.py

@@ -385,6 +385,14 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         help="Constrains effort on reasoning for reasoning models. Open AI o1 models only.",
         envvar="INSPECT_EVAL_REASONING_EFFORT",
     )
+    @click.option(
+        "--reasoning-history/--no-reasoning-history",
+        type=bool,
+        is_flag=True,
+        default=True,
+        help="Include reasoning in chat message history sent to generate.",
+        envvar="INSPECT_EVAL_REASONING_HISTORY",
+    )
     @click.option(
         "--log-format",
         type=click.Choice(["eval", "json"], case_sensitive=False),
@@ -444,6 +452,7 @@ def eval_command(
     max_tool_output: int | None,
     cache_prompt: str | None,
     reasoning_effort: str | None,
+    reasoning_history: bool | None,
     message_limit: int | None,
     token_limit: int | None,
     time_limit: int | None,
@@ -603,6 +612,7 @@ def eval_set_command(
     max_tool_output: int | None,
     cache_prompt: str | None,
     reasoning_effort: str | None,
+    reasoning_history: bool | None,
     message_limit: int | None,
     token_limit: int | None,
     time_limit: int | None,
@@ -841,6 +851,9 @@ def config_from_locals(locals: dict[str, Any]) -> GenerateConfigArgs:
             if key == "internal_tools":
                 if value is not False:
                     value = None
+            if key == "reasoning_history":
+                if value is not False:
+                    value = None
             config[key] = value  # type: ignore
     return config
 

inspect_ai/_cli/main.py

@@ -53,7 +53,7 @@ inspect.add_command(trace_command)
 def main() -> None:
     set_exception_hook()
     init_dotenv()
-    inspect(auto_envvar_prefix="INSPECT")
+    inspect(auto_envvar_prefix="INSPECT")  # pylint: disable=no-value-for-parameter
 
 
 if __name__ == "__main__":

inspect_ai/_cli/trace.py

@@ -109,11 +109,13 @@ def anomolies_command(trace_file: str | None, filter: str | None, all: bool) ->
     canceled_actions: dict[str, ActionTraceRecord] = {}
     error_actions: dict[str, ActionTraceRecord] = {}
     timeout_actions: dict[str, ActionTraceRecord] = {}
+    start_trace: ActionTraceRecord | None = None
 
     def action_started(trace: ActionTraceRecord) -> None:
         running_actions[trace.trace_id] = trace
 
     def action_completed(trace: ActionTraceRecord) -> ActionTraceRecord:
+        nonlocal start_trace
         start_trace = running_actions.get(trace.trace_id)
         if start_trace:
             del running_actions[trace.trace_id]
@@ -122,14 +124,20 @@ def anomolies_command(trace_file: str | None, filter: str | None, all: bool) ->
             raise RuntimeError(f"Expected {trace.trace_id} in action dictionary.")
 
     def action_failed(trace: ActionTraceRecord) -> None:
+        nonlocal start_trace
         if all:
+            assert start_trace
             error_actions[start_trace.trace_id] = trace
 
     def action_canceled(trace: ActionTraceRecord) -> None:
+        nonlocal start_trace
+        assert start_trace
         canceled_actions[start_trace.trace_id] = trace
 
     def action_timeout(trace: ActionTraceRecord) -> None:
+        nonlocal start_trace
         if all:
+            assert start_trace
             timeout_actions[start_trace.trace_id] = trace
 
     for trace in traces:

inspect_ai/_cli/view.py

@@ -63,6 +63,10 @@ def start(
     INSPECT_VIEW_AUTHORIZATION_TOKEN = "INSPECT_VIEW_AUTHORIZATION_TOKEN"
     authorization = os.environ.get(INSPECT_VIEW_AUTHORIZATION_TOKEN, None)
     if authorization:
+        # this indicates we are in vscode -- we want to set the log level to HTTP
+        # in vscode, updated versions of the extension do this but we set it
+        # manually here as a temporary bridge for running against older versions
+        common["log_level"] = "HTTP"
         del os.environ[INSPECT_VIEW_AUTHORIZATION_TOKEN]
         os.unsetenv(INSPECT_VIEW_AUTHORIZATION_TOKEN)
 

inspect_ai/_display/core/active.py

@@ -10,6 +10,8 @@ from ..rich.display import RichDisplay
 from ..textual.display import TextualDisplay
 from .display import Display, TaskScreen
 
+_active_display: Display | None = None
+
 
 def display() -> Display:
     global _active_display
@@ -28,9 +30,6 @@ def display() -> Display:
     return _active_display
 
 
-_active_display: Display | None = None
-
-
 def task_screen() -> TaskScreen:
     screen = _active_task_screen.get(None)
     if screen is None:

inspect_ai/_display/textual/widgets/transcript.py

@@ -15,6 +15,7 @@ from inspect_ai._util.transcript import (
     set_transcript_markdown_options,
     transcript_function,
     transcript_markdown,
+    transcript_reasoning,
     transcript_separator,
 )
 from inspect_ai.log._samples import ActiveSample
@@ -33,7 +34,11 @@ from inspect_ai.log._transcript import (
     SubtaskEvent,
     ToolEvent,
 )
-from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+from inspect_ai.model._chat_message import (
+    ChatMessage,
+    ChatMessageAssistant,
+    ChatMessageUser,
+)
 from inspect_ai.model._render import messages_preceding_assistant
 from inspect_ai.tool._tool import ToolResult
 from inspect_ai.tool._tool_transcript import transcript_tool_call
@@ -171,8 +176,8 @@ def render_model_event(event: ModelEvent) -> EventDisplay:
     # content
     content: list[RenderableType] = []
 
-    def append_message(message: ChatMessage, text: str | None = None) -> None:
-        content.extend(render_message(message, text))
+    def append_message(message: ChatMessage) -> None:
+        content.extend(render_message(message))
 
     # render preceding messages
     preceding = messages_preceding_assistant(event.input)
@@ -309,16 +314,17 @@ def render_as_json(json: Any) -> RenderableType:
     )
 
 
-def render_message(
-    message: ChatMessage, text: str | None = None
-) -> list[RenderableType]:
+def render_message(message: ChatMessage) -> list[RenderableType]:
     content: list[RenderableType] = [
         Text(message.role.capitalize(), style="bold"),
         Text(),
     ]
-    text = text or message.text
-    if text:
-        content.extend([transcript_markdown(text.strip(), escape=True)])
+
+    if isinstance(message, ChatMessageAssistant) and message.reasoning:
+        content.extend(transcript_reasoning(message.reasoning))
+
+    if message.text:
+        content.extend([transcript_markdown(message.text.strip(), escape=True)])
     return content
 
 

inspect_ai/_eval/eval.py

@@ -200,6 +200,10 @@ def eval(
     )
 
 
+# single call to eval_async at a time
+_eval_async_running = False
+
+
 async def eval_async(
     tasks: Tasks,
     model: str | Model | list[str] | list[Model] | None = None,
@@ -461,10 +465,6 @@ async def eval_async(
     return logs
 
 
-# single call to eval_async at a time
-_eval_async_running = False
-
-
 def eval_retry(
     tasks: str | EvalLogInfo | EvalLog | list[str] | list[EvalLogInfo] | list[EvalLog],
     log_level: str | None = None,

inspect_ai/_eval/evalset.py

@@ -43,6 +43,12 @@ from .task.task import PreviousTask, Task
 logger = logging.getLogger(__name__)
 
 
+class Log(NamedTuple):
+    info: EvalLogInfo
+    header: EvalLog
+    task_identifier: str
+
+
 def eval_set(
     tasks: Tasks,
     log_dir: str,
@@ -452,12 +458,6 @@ def return_last_value(retry_state: RetryCallState) -> list[EvalLog]:
         return []
 
 
-class Log(NamedTuple):
-    info: EvalLogInfo
-    header: EvalLog
-    task_identifier: str
-
-
 # list all eval logs
 def list_all_eval_logs(log_dir: str) -> list[Log]:
     log_files = list_eval_logs(log_dir)

inspect_ai/_eval/task/error.py

@@ -8,28 +8,24 @@ class SampleErrorHandler:
         self.fail_on_error = True if fail_on_error is None else fail_on_error
         self.total_samples = float(total_samples)
 
-    def __call__(self, ex: BaseException) -> EvalError:
+    def __call__(self, ex: BaseException) -> tuple[EvalError, BaseException | None]:
         # increment error count
         self.error_count += 1
 
         # create error (we may return it)
-        def sample_error() -> EvalError:
-            return eval_error(ex, type(ex), ex, ex.__traceback__)
+        def sample_error(
+            *, raise_error: bool
+        ) -> tuple[EvalError, BaseException | None]:
+            return eval_error(
+                ex, type(ex), ex, ex.__traceback__
+            ), ex if raise_error else None
 
         # check against limits
         if isinstance(self.fail_on_error, bool):
-            if self.fail_on_error:
-                raise ex
-            else:
-                return sample_error()
+            return sample_error(raise_error=self.fail_on_error)
         else:
             if self.fail_on_error < 1:
                 max_errors = self.fail_on_error * self.total_samples
-                if self.error_count >= max_errors:
-                    raise ex
-                else:
-                    return sample_error()
-            elif self.error_count >= self.fail_on_error:
-                raise ex
+                return sample_error(raise_error=self.error_count >= max_errors)
             else:
-                return sample_error()
+                return sample_error(raise_error=self.error_count >= self.fail_on_error)

inspect_ai/_eval/task/run.py

@@ -496,7 +496,7 @@ async def task_run_sample(
     logger: TaskLogger | None,
     log_images: bool,
     sample_source: EvalSampleSource | None,
-    sample_error: Callable[[BaseException], EvalError],
+    sample_error: SampleErrorHandler,
     sample_complete: Callable[[dict[str, SampleScore]], None],
     fails_on_error: bool,
     time_limit: int | None,
@@ -548,12 +548,12 @@ async def task_run_sample(
     )
 
     # helper to handle exceptions (will throw if we've exceeded the limit)
-    def handle_error(ex: BaseException) -> EvalError:
+    def handle_error(ex: BaseException) -> tuple[EvalError, BaseException | None]:
         err = sample_error(ex)
         py_logger.warning(
             f"Sample error (id: {sample.id}, epoch: {state.epoch}): {exception_message(ex)})"
         )
-        transcript()._event(ErrorEvent(error=err))
+        transcript()._event(ErrorEvent(error=err[0]))
         return err
 
     # solver loop
@@ -572,6 +572,7 @@ async def task_run_sample(
         ) as active,
     ):
         error: EvalError | None = None
+        raise_error: BaseException | None = None
         results: dict[str, SampleScore] = {}
         try:
             async with sandboxenv_cm:
@@ -640,7 +641,7 @@ async def task_run_sample(
                                 state = sample_state() or state
                             case "error":
                                 # default error handling
-                                error = handle_error(ex)
+                                error, raise_error = handle_error(ex)
 
                     else:
                         raise
@@ -660,7 +661,7 @@ async def task_run_sample(
                     state.completed = True
 
                 except BaseException as ex:
-                    error = handle_error(ex)
+                    error, raise_error = handle_error(ex)
 
                 # set timeout for scoring. if the original timeout was hit we still
                 # want to provide opportunity for scoring, but we don't necessarily
@@ -710,6 +711,9 @@ async def task_run_sample(
                                     results[name] = SampleScore(
                                         score=score, sample_id=state.sample_id
                                     )
+                                    transcript()._event(
+                                        ScoreEvent(score=score, target=sample.target)
+                                    )
 
                             # propagate results into scores
                             state.scores = {k: v.score for k, v in results.items()}
@@ -737,11 +741,10 @@ async def task_run_sample(
                         )
 
                     # handle error (this will throw if we've exceeded the limit)
-                    error = handle_error(ex)
+                    error, raise_error = handle_error(ex)
 
-        # handle sandboxenv init errors
         except Exception as ex:
-            error = handle_error(ex)
+            error, raise_error = handle_error(ex)
 
         # complete the sample
         progress(SAMPLE_TOTAL_PROGRESS_UNITS)
@@ -772,6 +775,8 @@ async def task_run_sample(
             if results is not None:
                 sample_complete(results)
             return results
+        elif raise_error:
+            raise raise_error
         else:
             return None
 

inspect_ai/_util/hash.py

@@ -3,7 +3,7 @@ import mmh3
 
 def mm3_hash(message: str) -> str:
     # Generate the 128-bit hash as two 64-bit integers
-    h1, h2 = mmh3.hash64(message.encode("utf-8"))
+    h1, h2 = mmh3.hash64(message.encode("utf-8"))  # pylint: disable=E0633
 
     # Convert to unsigned integers and then to hexadecimal
     return f"{h1 & 0xFFFFFFFFFFFFFFFF:016x}{h2 & 0xFFFFFFFFFFFFFFFF:016x}"

inspect_ai/_util/transcript.py

@@ -111,6 +111,17 @@ def transcript_panel(
     )
 
 
+def transcript_reasoning(reasoning: str) -> list[RenderableType]:
+    content: list[RenderableType] = []
+    content.append(
+        transcript_markdown(
+            f"**<think>**  \n{reasoning}  \n**</think>**\n\n", escape=True
+        )
+    )
+    content.append(Text())
+    return content
+
+
 def transcript_separator(title: str, color: str) -> RenderableType:
     return Rule(title=title, style=f"{color} bold", align="center", end="\n\n")
 

inspect_ai/_view/www/.vscode/extensions.json

@@ -0,0 +1,3 @@
+{
+  "recommendations": ["esbenp.prettier-vscode", "dbaeumer.vscode-eslint"]
+}

inspect_ai/_view/www/.vscode/settings.json

@@ -0,0 +1,8 @@
+{
+  "editor.formatOnSave": true,
+  "editor.defaultFormatter": "esbenp.prettier-vscode",
+  "editor.codeActionsOnSave": {
+    "source.organizeImports": "explicit",
+    "source.fixAll": "explicit"
+  }
+}

inspect_ai/_view/www/App.css

@@ -9,12 +9,27 @@
   --inspect-input-border: var(--bs-light-border-subtle);
   --inspect-diff-add-color: #dafbe1;
   --inspect-diff-remove-color: #ffebe9;
-  --inspect-inactive-selection-background: var(--vscode-editor-inactiveSelectionBackground, #d9d9d9);
-  --inspect-active-selection-background: var(--vscode-editor-selectionBackground, #d7d4f0);
+  --inspect-inactive-selection-background: var(
+    --vscode-editor-inactiveSelectionBackground,
+    #d9d9d9
+  );
+  --inspect-active-selection-background: var(
+    --vscode-editor-selectionBackground,
+    #d7d4f0
+  );
   --inspect-focus-border-color: #86b7fe;
   --inspect-focus-border-shadow: 0 0 0 0.25rem rgba(var(--bs-primary-rgb), 0.25);
   --inspect-focus-border-gray-color: #808080;
   --inspect-focus-border-gray-shadow: 0 0 0 0.25rem rgba(48, 48, 48, 0.25);
+
+  /* Inspect Font Sizes */
+  --inspect-font-size-title: 1.5rem;
+  --inspect-font-size-title-secondary: 1.3rem;
+  --inspect-font-size-larger: 1.1rem;
+  --inspect-font-size-large: 1rem;
+  --inspect-font-size-base: 0.9rem;
+  --inspect-font-size-small: 0.8rem;
+  --inspect-font-size-smaller: 0.8rem;
 }
 
 body:not([class^="vscode-"]) button {
@@ -47,6 +62,61 @@ body[class^="vscode-"] .app-main-grid {
   grid-template-rows: max-content max-content 1fr;
 }
 
+/* Inspect Text Styles */
+.text-style-label {
+  text-transform: uppercase;
+}
+
+.text-style-secondary {
+  color: var(--bs-secondary);
+}
+
+.text-style-tertiary {
+  color: var(--bs-tertiary-color);
+}
+
+/* Inspect Font Size Styles */
+.text-size-title {
+  font-size: var(--inspect-font-size-title);
+}
+
+.text-size-title-secondary {
+  font-size: var(--inspect-font-size-title-secondary);
+}
+
+.text-size-larger {
+  font-size: var(--inspect-font-size-larger);
+}
+
+.text-size-large {
+  font-size: var(--inspect-font-size-large);
+}
+
+.text-size-base {
+  font-size: var(--inspect-font-size-base);
+}
+
+.text-size-small {
+  font-size: var(--inspect-font-size-small);
+}
+
+.text-size-smaller {
+  font-size: var(--inspect-font-size-smaller);
+}
+
+.text-truncate {
+  white-space: nowrap;
+  text-overflow: ellipsis;
+  overflow: hidden;
+}
+
+.three-line-clamp {
+  display: -webkit-box;
+  -webkit-line-clamp: 3;
+  -webkit-box-orient: vertical;
+  overflow: hidden;
+}
+
 body[class^="vscode-"] {
   --bs-border-radius: 0;
   --bs-border-radius-lg: 0;
@@ -87,7 +157,7 @@ html.vscode {
 
 html.vscode .sample-input {
   line-height: 1.3em;
-  -webkit-line-clamp: 4 !important
+  -webkit-line-clamp: 4 !important;
 }
 
 body[class^="vscode-"] .modal-backdrop {
@@ -276,7 +346,7 @@ body {
 }
 
 @media (max-width: 575px) {
-  .tab-tools  select {
+  .tab-tools select {
     width: 50px;
   }
 }
@@ -312,12 +382,6 @@ body {
   font-size: 1.5em;
 }
 
-.sidebar {
-  --bs-offcanvas-width: var(--sidebar-width);
-  width: var(--sidebar-width);
-  overflow-y: auto;
-}
-
 .nav-link.active {
   border-bottom-width: 0 !important;
 }
@@ -644,7 +708,7 @@ table.table.table-sm td {
 
 .tab-tools .btn {
   font-size: 0.7rem;
-  padding: 0.4em 0.8em;
+  padding: 0.2em 0.8em;
 }
 
 .tab-tools {
@@ -724,7 +788,7 @@ table.table.table-sm td {
 }
 
 @keyframes moveLeftToRight {
-from {
+  from {
     margin-left: 0;
   }
   to {
@@ -760,7 +824,6 @@ pre[class*="language-"].tool-output {
 
 /* lightbox styles */
 
-
 .lightbox-overlay .close-button,
 .lightbox-overlay .nav-button {
   /* Hide by default */
@@ -868,38 +931,38 @@ ul.jsondiffpatch-delta {
   vertical-align: top;
 }
 .jsondiffpatch-property-name:after {
-  content: ': ';
+  content: ": ";
 }
 .jsondiffpatch-child-node-type-array > .jsondiffpatch-property-name:after {
-  content: ': [';
+  content: ": [";
 }
 .jsondiffpatch-child-node-type-array:after {
-  content: '],';
+  content: "],";
 }
 div.jsondiffpatch-child-node-type-array:before {
-  content: '[';
+  content: "[";
 }
 div.jsondiffpatch-child-node-type-array:after {
-  content: ']';
+  content: "]";
 }
 .jsondiffpatch-child-node-type-object > .jsondiffpatch-property-name:after {
-  content: ': {';
+  content: ": {";
 }
 .jsondiffpatch-child-node-type-object:after {
-  content: '},';
+  content: "},";
 }
 div.jsondiffpatch-child-node-type-object:before {
-  content: '{';
+  content: "{";
 }
 div.jsondiffpatch-child-node-type-object:after {
-  content: '}';
+  content: "}";
 }
 .jsondiffpatch-value pre:after {
-  content: ',';
+  content: ",";
 }
 li:last-child > .jsondiffpatch-value pre:after,
 .jsondiffpatch-modified > .jsondiffpatch-left-value pre:after {
-  content: '';
+  content: "";
 }
 .jsondiffpatch-modified .jsondiffpatch-value {
   display: inline-block;
@@ -916,7 +979,7 @@ li:last-child > .jsondiffpatch-value pre:after,
   color: #888;
 }
 .jsondiffpatch-moved .jsondiffpatch-moved-destination:before {
-  content: ' => ';
+  content: " => ";
 }
 ul.jsondiffpatch-textdiff {
   padding: 0;
@@ -930,7 +993,7 @@ ul.jsondiffpatch-textdiff {
   display: inline-block;
 }
 .jsondiffpatch-textdiff-line-number:after {
-  content: ',';
+  content: ",";
 }
 .jsondiffpatch-error {
   background: red;
@@ -976,14 +1039,14 @@ ul.jsondiffpatch-textdiff {
   padding: 1em;
   margin: 0.5em 0;
   overflow: auto;
-  border: 0.3em solid #7a6651;
+  /* border: 0.3em solid #7a6651; */
   border-radius: 0.5em;
   box-shadow: 1px 1px 0.5em #000 inset;
 }
 .vscode-dark :not(pre) > code[class*="language-"] {
   padding: 0.15em 0.2em 0.05em;
   border-radius: 0.3em;
-  border: 0.13em solid #7a6651;
+  /* border: 0.13em solid #7a6651; */
   box-shadow: 1px 1px 0.3em -0.1em #000 inset;
   white-space: normal;
 }
@@ -1045,4 +1108,4 @@ ul.jsondiffpatch-textdiff {
 .vscode-dark .token.deleted {
   color: red;
 }
-/* END PrismJS */
+/* END PrismJS */