inspect-ai 0.3.104__py3-none-any.whl → 0.3.106__py3-none-any.whl

inspect_ai/_eval/context.py

@@ -1,4 +1,7 @@
+from anyio.abc import TaskGroup
+
 from inspect_ai._util.dotenv import init_dotenv
+from inspect_ai._util.eval_task_group import init_eval_task_group
 from inspect_ai._util.hooks import init_hooks
 from inspect_ai._util.logger import init_logger
 from inspect_ai.approval._apply import have_tool_approval, init_tool_approval
@@ -19,6 +22,7 @@ def init_eval_context(
     log_level: str | None,
     log_level_transcript: str | None,
     max_subprocesses: int | None = None,
+    task_group: TaskGroup | None = None,
 ) -> None:
     init_dotenv()
     init_logger(log_level, log_level_transcript)
@@ -27,6 +31,7 @@ def init_eval_context(
     init_hooks()
     init_active_samples()
     init_human_approval_manager()
+    init_eval_task_group(task_group)
 
 
 def init_task_context(

inspect_ai/_eval/eval.py

@@ -4,11 +4,15 @@ import sys
 from pathlib import Path
 from typing import Any, Literal, cast
 
+import anyio
+from anyio.abc import TaskGroup
+
 from inspect_ai._eval.task.task import resolve_model_roles
 from inspect_ai._util.notgiven import NOT_GIVEN, NotGiven
 from inspect_ai.agent._agent import Agent, is_agent
 from inspect_ai.agent._as_solver import as_solver
 from inspect_ai.log._model import model_roles_config_to_model_roles
+from inspect_ai.util._anyio import inner_exception
 
 if sys.version_info < (3, 11):
     from exceptiongroup import ExceptionGroup
@@ -359,6 +363,112 @@ async def eval_async(
     Returns:
         List of EvalLog (one for each task)
     """
+    result: list[EvalLog] | None = None
+
+    async def run(tg: TaskGroup) -> None:
+        try:
+            nonlocal result
+            result = await _eval_async_inner(
+                tg=tg,
+                tasks=tasks,
+                model=model,
+                model_base_url=model_base_url,
+                model_args=model_args,
+                model_roles=model_roles,
+                task_args=task_args,
+                sandbox=sandbox,
+                sandbox_cleanup=sandbox_cleanup,
+                solver=solver,
+                tags=tags,
+                metadata=metadata,
+                approval=approval,
+                log_level=log_level,
+                log_level_transcript=log_level_transcript,
+                log_dir=log_dir,
+                log_format=log_format,
+                limit=limit,
+                sample_id=sample_id,
+                epochs=epochs,
+                fail_on_error=fail_on_error,
+                retry_on_error=retry_on_error,
+                debug_errors=debug_errors,
+                message_limit=message_limit,
+                token_limit=token_limit,
+                time_limit=time_limit,
+                working_limit=working_limit,
+                max_samples=max_samples,
+                max_tasks=max_tasks,
+                max_subprocesses=max_subprocesses,
+                max_sandboxes=max_sandboxes,
+                log_samples=log_samples,
+                log_realtime=log_realtime,
+                log_images=log_images,
+                log_buffer=log_buffer,
+                log_shared=log_shared,
+                log_header_only=log_header_only,
+                score=score,
+                score_display=score_display,
+                **kwargs,
+            )
+        finally:
+            tg.cancel_scope.cancel()
+
+    try:
+        async with anyio.create_task_group() as tg:
+            tg.start_soon(run, tg)
+    except Exception as ex:
+        raise inner_exception(ex)
+    except anyio.get_cancelled_exc_class():
+        # Cancelled exceptions are expected and handled by _eval_async_inner
+        pass
+
+    assert result is not None, "Eval async did not return a result."
+
+    return result
+
+
+async def _eval_async_inner(
+    tg: TaskGroup,
+    tasks: Tasks,
+    model: str | Model | list[str] | list[Model] | None | NotGiven = NOT_GIVEN,
+    model_base_url: str | None = None,
+    model_args: dict[str, Any] | str = dict(),
+    model_roles: dict[str, str | Model] | None = None,
+    task_args: dict[str, Any] | str = dict(),
+    sandbox: SandboxEnvironmentType | None = None,
+    sandbox_cleanup: bool | None = None,
+    solver: Solver | SolverSpec | Agent | list[Solver] | None = None,
+    tags: list[str] | None = None,
+    metadata: dict[str, Any] | None = None,
+    approval: str | list[ApprovalPolicy] | ApprovalPolicyConfig | None = None,
+    log_level: str | None = None,
+    log_level_transcript: str | None = None,
+    log_dir: str | None = None,
+    log_format: Literal["eval", "json"] | None = None,
+    limit: int | tuple[int, int] | None = None,
+    sample_id: str | int | list[str] | list[int] | list[str | int] | None = None,
+    epochs: int | Epochs | None = None,
+    fail_on_error: bool | float | None = None,
+    retry_on_error: int | None = None,
+    debug_errors: bool | None = None,
+    message_limit: int | None = None,
+    token_limit: int | None = None,
+    time_limit: int | None = None,
+    working_limit: int | None = None,
+    max_samples: int | None = None,
+    max_tasks: int | None = None,
+    max_subprocesses: int | None = None,
+    max_sandboxes: int | None = None,
+    log_samples: bool | None = None,
+    log_realtime: bool | None = None,
+    log_images: bool | None = None,
+    log_buffer: int | None = None,
+    log_shared: bool | int | None = None,
+    log_header_only: bool | None = None,
+    score: bool = True,
+    score_display: bool | None = None,
+    **kwargs: Unpack[GenerateConfigArgs],
+) -> list[EvalLog]:
     # only a single call to eval_async can be active at a time, this used
     # to be due to running tasks switching to the task's directory, however
     # that feature no longer exists so we may be able to revisit this
@@ -387,6 +497,7 @@ async def eval_async(
             max_subprocesses=max_subprocesses,
             log_level=log_level,
             log_level_transcript=log_level_transcript,
+            task_group=tg,
             **kwargs,
         )
 
@@ -934,10 +1045,11 @@ def eval_init(
     max_subprocesses: int | None = None,
     log_level: str | None = None,
     log_level_transcript: str | None = None,
+    task_group: TaskGroup | None = None,
     **kwargs: Unpack[GenerateConfigArgs],
 ) -> list[Model]:
     # init eval context
-    init_eval_context(log_level, log_level_transcript, max_subprocesses)
+    init_eval_context(log_level, log_level_transcript, max_subprocesses, task_group)
 
     # resolve model and task args
     model_args = resolve_args(model_args)

inspect_ai/_eval/evalset.py

@@ -578,7 +578,7 @@ def task_identifier(task: ResolvedTask | EvalLog) -> str:
     else:
         task_file = task.eval.task_file or ""
         task_name = task.eval.task
-        task_args = task.eval.task_args
+        task_args = task.eval.task_args_passed
         model = str(task.eval.model)
         model_roles = task.eval.model_roles or {}
 

inspect_ai/_eval/task/run.py

@@ -10,6 +10,7 @@ from pathlib import PurePath
 from typing import Callable, Literal
 
 import anyio
+from anyio.abc import TaskGroup
 from typing_extensions import Unpack
 
 from inspect_ai._display import (
@@ -306,37 +307,57 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
                     task.metrics,
                 )
 
+                async def run_sample(
+                    sample: Sample, state: TaskState
+                ) -> dict[str, SampleScore] | None:
+                    result: dict[str, SampleScore] | None = None
+
+                    async def run(tg: TaskGroup) -> None:
+                        try:
+                            nonlocal result
+                            result = await task_run_sample(
+                                tg=tg,
+                                task_name=task.name,
+                                log_location=profile.log_location,
+                                sample=sample,
+                                state=state,
+                                sandbox=sandbox,
+                                max_sandboxes=config.max_sandboxes,
+                                sandbox_cleanup=sandbox_cleanup,
+                                plan=plan,
+                                scorers=scorers,
+                                generate=generate,
+                                progress=progress,
+                                logger=logger if log_samples else None,
+                                log_images=log_images,
+                                sample_source=sample_source,
+                                sample_error=sample_error_handler,
+                                sample_complete=sample_complete,
+                                fails_on_error=(
+                                    config.fail_on_error is None
+                                    or config.fail_on_error is True
+                                ),
+                                retry_on_error=config.retry_on_error or 0,
+                                error_retries=[],
+                                time_limit=config.time_limit,
+                                working_limit=config.working_limit,
+                                semaphore=sample_semaphore,
+                            )
+                        finally:
+                            tg.cancel_scope.cancel()
+
+                    async with anyio.create_task_group() as tg:
+                        tg.start_soon(run, tg)
+
+                    return result
+
                 sample_results = await tg_collect(
                     [
-                        functools.partial(
-                            task_run_sample,
-                            task_name=task.name,
-                            log_location=profile.log_location,
-                            sample=sample,
-                            state=state,
-                            sandbox=sandbox,
-                            max_sandboxes=config.max_sandboxes,
-                            sandbox_cleanup=sandbox_cleanup,
-                            plan=plan,
-                            scorers=scorers,
-                            generate=generate,
-                            progress=progress,
-                            logger=logger if log_samples else None,
-                            log_images=log_images,
-                            sample_source=sample_source,
-                            sample_error=sample_error_handler,
-                            sample_complete=sample_complete,
-                            fails_on_error=(
-                                config.fail_on_error is None
-                                or config.fail_on_error is True
-                            ),
-                            retry_on_error=config.retry_on_error or 0,
-                            error_retries=[],
-                            time_limit=config.time_limit,
-                            working_limit=config.working_limit,
-                            semaphore=sample_semaphore,
+                        functools.partial(run_sample, sample, state)
+                        for (sample, state) in zip(
+                            samples,
+                            states,
                         )
-                        for (sample, state) in zip(samples, states)
                     ]
                 )
 
@@ -492,6 +513,7 @@ def update_metrics_display_fn(
 
 async def task_run_sample(
     *,
+    tg: TaskGroup,
     task_name: str,
     log_location: str,
     sample: Sample,
@@ -611,12 +633,14 @@ async def task_run_sample(
             working_limit=working_limit,
             fails_on_error=fails_on_error or (retry_on_error > 0),
             transcript=sample_transcript,
+            tg=tg,
         ) as active,
     ):
         start_time: float | None = None
         error: EvalError | None = None
         raise_error: BaseException | None = None
         results: dict[str, SampleScore] = {}
+        limit: EvalSampleLimit | None = None
         try:
             # begin init
             init_span = span("init", type="init")
@@ -704,9 +728,17 @@ async def task_run_sample(
                         # handle the cancel exception
                         raise
 
-                except (LimitExceededError, TerminateSampleError):
+                except LimitExceededError as ex:
+                    # capture most recent state for scoring
+                    state = sample_state() or state
+                    limit = EvalSampleLimit(
+                        type=ex.type, limit=ex.limit if ex.limit is not None else -1
+                    )
+
+                except TerminateSampleError:
                     # capture most recent state for scoring
                     state = sample_state() or state
+                    limit = EvalSampleLimit(type="operator", limit=1)
 
                 except BaseException as ex:
                     error, raise_error = handle_error(ex)
@@ -815,6 +847,7 @@ async def task_run_sample(
                     state=state,
                     scores=results,
                     error=error,
+                    limit=limit,
                     error_retries=error_retries,
                     log_images=log_images,
                 )
@@ -854,6 +887,7 @@ async def task_run_sample(
             time_limit=time_limit,
             working_limit=working_limit,
             semaphore=semaphore,
+            tg=tg,
         )
 
     # no error
@@ -879,6 +913,7 @@ async def log_sample(
     state: TaskState,
     scores: dict[str, SampleScore],
     error: EvalError | None,
+    limit: EvalSampleLimit | None,
     error_retries: list[EvalError],
     log_images: bool,
 ) -> None:
@@ -894,15 +929,6 @@ async def log_sample(
     # compute total time if we can
     total_time = time.monotonic() - start_time if start_time is not None else None
 
-    # if a limit was hit, note that in the Eval Sample
-    limit = None
-    for e in transcript().events:
-        if e.event == "sample_limit":
-            limit = EvalSampleLimit(
-                type=e.type, limit=e.limit if e.limit is not None else -1
-            )
-            break
-
     eval_sample = EvalSample(
         id=id,
         epoch=state.epoch,

inspect_ai/_util/eval_task_group.py

@@ -0,0 +1,15 @@
+from anyio.abc import TaskGroup
+
+_eval_task_group: TaskGroup | None = None
+
+
+def init_eval_task_group(tg: TaskGroup | None) -> None:
+    global _eval_task_group
+    _eval_task_group = tg
+
+
+def eval_task_group() -> TaskGroup:
+    global _eval_task_group
+    if _eval_task_group is None:
+        raise RuntimeError("Task group has not been initialized")
+    return _eval_task_group

inspect_ai/_view/server.py

@@ -155,6 +155,23 @@ def view_server(
                 body=samples.model_dump_json(), headers={"ETag": samples.etag}
             )
 
+    @routes.get("/api/log-message")
+    async def api_log_message(request: web.Request) -> web.Response:
+        # log file requested
+        file = query_param_required("log_file", request, str)
+
+        file = urllib.parse.unquote(file)
+        validate_log_file_request(file)
+
+        # message to log
+        message = query_param_required("message", request, str)
+
+        # log the message
+        logger.warning(f"[CLIENT MESSAGE] ({file}): {message}")
+
+        # respond
+        return web.Response(status=204)
+
     @routes.get("/api/pending-sample-data")
     async def api_sample_events(request: web.Request) -> web.Response:
         # log file requested

inspect_ai/_view/www/dist/assets/index.css

@@ -15655,18 +15655,40 @@ pre[class*="language-"] {
   background-color: var(--bs-light-border-subtle);
   margin-top: -1px;
 }
-._message_17kai_1 {
+._keyPairContainer_1ltuo_1 {
+  display: grid;
+  grid-template-columns: max-content auto;
+  column-gap: 0.5em;
+  padding-top: 4px;
+  padding-bottom: 4px;
+  border-bottom: solid 1px var(--bs-border-color);
+}
+
+._key_1ltuo_1 {
+  display: grid;
+  grid-template-columns: 1em auto;
+  cursor: pointer;
+}
+
+._pre_1ltuo_16 {
+  margin-bottom: 0;
+}
+
+._treeIcon_1ltuo_20 {
+  margin-top: -3px;
+}
+._message_1ivu3_1 {
   font-weight: 300;
   margin-left: 0;
   margin-right: 0;
   white-space: normal;
 }
 
-._systemRole_17kai_8 {
+._systemRole_1ivu3_8 {
   opacity: 0.7;
 }
 
-._messageGrid_17kai_12 {
+._messageGrid_1ivu3_12 {
   display: grid;
   grid-template-columns: max-content max-content max-content;
   column-gap: 0.3em;
@@ -15674,24 +15696,28 @@ pre[class*="language-"] {
   margin-bottom: 0.3em;
 }
 
-._messageContents_17kai_20 {
+._messageContents_1ivu3_20 {
   margin-left: 0;
   padding-bottom: 0;
 }
 
-._messageContents_17kai_20._indented_17kai_25 {
+._messageContents_1ivu3_20._indented_1ivu3_25 {
   margin-left: 0rem;
 }
 
-._copyLink_17kai_29 {
+._copyLink_1ivu3_29 {
   opacity: 0;
   padding-left: 0;
   padding-right: 2em;
 }
 
-._copyLink_17kai_29:hover {
+._copyLink_1ivu3_29:hover {
   opacity: 1;
 }
+
+._metadataLabel_1ivu3_39 {
+  padding-top: 1em;
+}
 ._webSearch_1376z_1 {
   display: grid;
   grid-template-columns: max-content 1fr;
@@ -15702,28 +15728,6 @@ pre[class*="language-"] {
 ._query_1376z_8 {
   font-family: var(--bs-font-monospace);
 }
-._keyPairContainer_1ltuo_1 {
-  display: grid;
-  grid-template-columns: max-content auto;
-  column-gap: 0.5em;
-  padding-top: 4px;
-  padding-bottom: 4px;
-  border-bottom: solid 1px var(--bs-border-color);
-}
-
-._key_1ltuo_1 {
-  display: grid;
-  grid-template-columns: 1em auto;
-  cursor: pointer;
-}
-
-._pre_1ltuo_16 {
-  margin-bottom: 0;
-}
-
-._treeIcon_1ltuo_20 {
-  margin-top: -3px;
-}
 ._contentData_1sd1z_1 {
   border: solid var(--bs-light-border-subtle) 1px;
   padding: 0.5em;