inspect-ai 0.3.59__py3-none-any.whl → 0.3.61__py3-none-any.whl

inspect_ai/_cli/eval.py

@@ -314,12 +314,6 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         help="Sequences where the API will stop generating further tokens. The returned text will not contain the stop sequence.",
         envvar="INSPECT_EVAL_STOP_SEQS",
     )
-    @click.option(
-        "--suffix",
-        type=str,
-        help="The suffix that comes after a completion of inserted text. OpenAI only.",
-        envvar="INSPECT_EVAL_SUFFIX",
-    )
     @click.option(
         "--temperature",
         type=float,
@@ -439,7 +433,6 @@ def eval_command(
     logit_bias: str | None,
     seed: int | None,
     stop_seqs: str | None,
-    suffix: str | None,
     temperature: float | None,
     top_p: float | None,
     top_k: int | None,
@@ -599,7 +592,6 @@ def eval_set_command(
     logit_bias: str | None,
     seed: int | None,
     stop_seqs: str | None,
-    suffix: str | None,
     temperature: float | None,
     top_p: float | None,
     top_k: int | None,

inspect_ai/_display/textual/widgets/samples.py

@@ -413,7 +413,7 @@ class SampleToolbar(Horizontal):
         grid-columns: auto auto 1fr auto auto;
     }}
     SampleToolbar #{STATUS_GROUP} {{
-        min-width: 20;
+        width: 22;
     }}
     SampleToolbar Button {{
         margin-bottom: 1;

inspect_ai/_eval/eval.py

@@ -35,7 +35,12 @@ from inspect_ai.scorer._reducer import reducer_log_names
 from inspect_ai.solver._chain import chain
 from inspect_ai.solver._solver import Solver, SolverSpec
 from inspect_ai.util import SandboxEnvironmentType
-from inspect_ai.util._display import DisplayType, display_type, init_display_type
+from inspect_ai.util._display import (
+    DisplayType,
+    display_type,
+    display_type_initialized,
+    init_display_type,
+)
 
 from .context import init_eval_context
 from .loader import ResolvedTask, resolve_tasks
@@ -306,6 +311,10 @@ async def eval_async(
 
     _eval_async_running = True
 
+    # if we are called outside of eval() then set display type to "plain"
+    if not display_type_initialized():
+        init_display_type("plain")
+
     # resolve model and task args
     model_args = resolve_args(model_args)
     task_args = resolve_args(task_args)

inspect_ai/_eval/loader.py

@@ -1,5 +1,6 @@
 import ast
 import contextlib
+import inspect
 import os
 from dataclasses import dataclass, field
 from importlib.machinery import SourceFileLoader
@@ -9,11 +10,13 @@ from pathlib import Path
 from types import ModuleType
 from typing import Any, Callable, cast
 
+from typing_extensions import overload
+
 from inspect_ai._eval.task.util import task_file, task_run_dir
 from inspect_ai._util.decorator import parse_decorators
 from inspect_ai._util.error import PrerequisiteError
 from inspect_ai._util.logger import warn_once
-from inspect_ai._util.path import chdir_python
+from inspect_ai._util.path import chdir_python, cwd_relative_path
 from inspect_ai._util.registry import (
     RegistryInfo,
     is_registry_object,
@@ -23,6 +26,7 @@ from inspect_ai._util.registry import (
     registry_params,
 )
 from inspect_ai.model import Model, ModelName
+from inspect_ai.solver._bridge import bridge
 from inspect_ai.solver._solver import Solver, SolverSpec
 from inspect_ai.util import SandboxEnvironmentSpec, SandboxEnvironmentType
 from inspect_ai.util._sandbox.environment import resolve_sandbox_environment
@@ -334,6 +338,16 @@ def split_spec(spec: str) -> tuple[str, str | None]:
         return spec, None
 
 
+@overload
+def load_module(
+    module_path: Path, filter: Callable[[str], bool]
+) -> ModuleType | None: ...
+
+
+@overload
+def load_module(module_path: Path, filter: None = None) -> ModuleType: ...
+
+
 def load_module(
     module_path: Path, filter: Callable[[str], bool] | None = None
 ) -> ModuleType | None:
@@ -425,28 +439,74 @@ def solver_from_spec(spec: SolverSpec) -> Solver:
         else contextlib.nullcontext()
     )
 
+    # pretty solver name for error messages
+    pretty_solver_file = (
+        cwd_relative_path(solver_file.as_posix()) if solver_file else None
+    )
+
     with create_cm:
-        # if we have a file then we need to load it and (if required) determine the solver name
-        if solver_file is not None:
-            # load the module so that registry_create works
-            load_module(solver_file)
+        # if there is no solver file then just create from the registry by name
+        if solver_file is None:
+            if solver_name is None:
+                raise ValueError(f"Unable to resolve solver name from {spec.solver}")
+            return cast(Solver, registry_create("solver", solver_name, **spec.args))
 
-            # if there is no solver_name we need to discover the first @solver
+        # we do have a solver file
+        else:
+            # load the module and parse decorators
+            solver_module = load_module(solver_file)
+            decorators = parse_decorators(solver_file, "solver")
+
+            # if there is no solver_name see if we can discover it
             if solver_name is None:
-                solvers = parse_decorators(solver_file, "solver")
-                if len(solvers) == 0:
+                if len(decorators) == 1:
+                    # decorator based solver
+                    solver_name = decorators[0][0]
+                elif len(decorators) == 0:
+                    # see if we can find an agent based solver
+                    functions = [
+                        function
+                        for function in inspect.getmembers(
+                            solver_module, inspect.isfunction
+                        )
+                        if function[1].__module__ == solver_module.__name__
+                    ]
+                    agent_functions = [
+                        function
+                        for function in functions
+                        if "agent" in function[0] and not function[0].startswith("_")
+                    ]
+                    if len(agent_functions) == 1:
+                        # agent based solver
+                        solver_name = agent_functions[0][0]
+
+                    elif len(agent_functions) == 0:
+                        raise PrerequisiteError(
+                            f"The source file {pretty_solver_file} does not contain any @solver functions or agent functions."
+                        )
+                    else:
+                        raise PrerequisiteError(
+                            f"The source file {pretty_solver_file} has more than one agent function (qualify which agent using e.g. '{solver_file.name}@agent_fn')"
+                        )
+                else:
                     raise PrerequisiteError(
-                        f"The source file {solver_file.as_posix()} does not contain any @solver functions."
+                        f"The source file {pretty_solver_file} has more than one @solver function (qualify which solver using e.g. '{solver_file.name}y@solver_fn')"
                     )
-                if len(solvers) > 1:
-                    raise PrerequisiteError(
-                        f"The source file {solver_file.as_posix()} has more than one @solver function (qualify which solver using file.py@solver)"
-                    )
-                solver_name = solvers[0][0]
 
-        # make mypy happy and catch unexpected branching
-        if solver_name is None:
-            raise ValueError(f"Unable to resolve solver name from {spec.solver}")
+            # create decorator based solvers using the registry
+            if any(solver[0] == solver_name for solver in decorators):
+                return cast(Solver, registry_create("solver", solver_name, **spec.args))
 
-        solver = cast(Solver, registry_create("solver", solver_name, **spec.args))
-        return solver
+            # create agent based solvers by calling the function and wrapping it in bridge()
+            else:
+                agent_fn = getattr(solver_module, solver_name, None)
+                if inspect.isfunction(agent_fn):
+                    return bridge(agent_fn(**spec.args))
+                elif agent_fn is not None:
+                    raise PrerequisiteError(
+                        f"The object {solver_name} in file {pretty_solver_file} is not a Python function."
+                    )
+                else:
+                    raise PrerequisiteError(
+                        f"The function {solver_name} was not found in file {pretty_solver_file}."
+                    )

inspect_ai/_eval/registry.py

@@ -1,6 +1,7 @@
 import inspect
 import logging
 from copy import deepcopy
+from functools import wraps
 from pathlib import Path
 from typing import Any, Callable, TypeVar, cast, overload
 
@@ -125,6 +126,7 @@ def task(*args: Any, name: str | None = None, **attribs: Any) -> Any:
         params = list(inspect.signature(task_type).parameters.keys())
 
         # Create and return the wrapper function
+        @wraps(task_type)
         def wrapper(*w_args: Any, **w_kwargs: Any) -> Task:
             # Create the task
             task_instance = task_type(*w_args, **w_kwargs)
@@ -154,6 +156,10 @@ def task(*args: Any, name: str | None = None, **attribs: Any) -> Any:
             # Return the task instance
             return task_instance
 
+        # functools.wraps overrides the return type annotation of the inner function, so
+        # we explicitly set it again
+        wrapper.__annotations__["return"] = Task
+
         # Register the task and return the wrapper
         return task_register(
             task=cast(TaskType, wrapper), name=task_name, attribs=attribs, params=params

inspect_ai/_eval/score.py

@@ -5,7 +5,7 @@ from typing import Callable, cast
 from inspect_ai._display import display
 from inspect_ai._util.path import chdir_python
 from inspect_ai._util.platform import platform_init
-from inspect_ai._util.registry import registry_create
+from inspect_ai._util.registry import registry_create, registry_unqualified_name
 from inspect_ai.log import (
     EvalLog,
     EvalMetric,
@@ -185,6 +185,7 @@ async def run_score_task(
         results[scorer_name] = SampleScore(
             score=result,
             sample_id=state.sample_id,
+            scorer=registry_unqualified_name(scorer),
         )
 
     progress()

inspect_ai/_eval/task/generate.py

@@ -8,6 +8,7 @@ from inspect_ai.model import (
 )
 from inspect_ai.model._cache import epoch
 from inspect_ai.solver import TaskState
+from inspect_ai.solver._limit import SampleLimitExceededError
 from inspect_ai.tool import ToolFunction
 
 
@@ -21,45 +22,50 @@ async def task_generate(
     # track tool_choice (revert to "auto" after first forced call of a tool)
     tool_choice = state.tool_choice
 
-    while True:
-        # If we don't update the epoch here as we go, it's entirely possible
-        # we'd cache the same response for every single epoch, which would
-        # completely defeat the point!
-        epoch.set(state.epoch)
+    try:
+        while True:
+            # If we don't update the epoch here as we go, it's entirely possible
+            # we'd cache the same response for every single epoch, which would
+            # completely defeat the point!
+            epoch.set(state.epoch)
 
-        # call the model
-        state.output = await model.generate(
-            input=state.messages,
-            tools=state.tools,
-            tool_choice=tool_choice,
-            config=config,
-            cache=cache,
-        )
+            # call the model
+            state.output = await model.generate(
+                input=state.messages,
+                tools=state.tools,
+                tool_choice=tool_choice,
+                config=config,
+                cache=cache,
+            )
 
-        # append the assistant message
-        message = state.output.message
-        state.messages.append(message)
+            # append the assistant message
+            message = state.output.message
+            state.messages.append(message)
 
-        # check for completed
-        if state.completed:
-            return state
+            # check for completed
+            if state.completed:
+                return state
 
-        # resolve tool calls if necessary
-        if tool_calls != "none" and message.tool_calls:
-            # call tools and append messages to state
-            state.messages.extend(
-                await call_tools(message, state.tools, config.max_tool_output)
-            )
+            # resolve tool calls if necessary
+            if tool_calls != "none" and message.tool_calls:
+                # call tools and append messages to state
+                state.messages.extend(
+                    await call_tools(message, state.tools, config.max_tool_output)
+                )
 
-            # check for completed or only executing a single tool call
-            if state.completed or tool_calls == "single":
-                return state
+                # check for completed or only executing a single tool call
+                if state.completed or tool_calls == "single":
+                    return state
+
+                # if a tool_call was forced set tool_choice to 'auto'
+                # (otherwise it will get forced over and over again)
+                if isinstance(tool_choice, ToolFunction):
+                    tool_choice = "auto"
 
-            # if a tool_call was forced set tool_choice to 'auto'
-            # (otherwise it will get forced over and over again)
-            if isinstance(tool_choice, ToolFunction):
-                tool_choice = "auto"
+            # no tool calls or not resolving tool calls, we are done!
+            else:
+                return state
 
-        # no tool calls or not resolving tool calls, we are done!
-        else:
-            return state
+    # propagate current state along with sample limit exceeded
+    except SampleLimitExceededError as ex:
+        raise ex.with_state(state)

inspect_ai/_eval/task/results.py

@@ -65,11 +65,12 @@ def eval_results(
     # extract scorers info from scorers then create scorers info for any
     # scores not already accounted for by a scorer name
     scorers_info = [ScorerInfo.from_scorer(scorer) for scorer in (scorers or [])]
-    scorer_names = [info.name for info in scorers_info]
-    for name in set(key for sample_scores in scores for key in sample_scores):
-        if name not in scorer_names:
-            scorers_info.append(ScorerInfo.from_name(name))
-            scorer_names.append(name)
+    scorer_names = {info.name for info in scorers_info}
+    for sample_scores in scores:
+        for name, sample_score in sample_scores.items():
+            if sample_score.scorer is None and name not in scorer_names:
+                scorers_info.append(ScorerInfo.from_name(name))
+                scorer_names.add(name)
 
     # record scorer
     if len(scorers_info) > 0:

inspect_ai/_eval/task/run.py

@@ -27,8 +27,12 @@ from inspect_ai._util.constants import (
 from inspect_ai._util.datetime import iso_now
 from inspect_ai._util.error import exception_message
 from inspect_ai._util.hooks import send_telemetry
-from inspect_ai._util.registry import is_registry_object, registry_log_name
-from inspect_ai._util.timeouts import Timeout, timeout, timeout_at
+from inspect_ai._util.registry import (
+    is_registry_object,
+    registry_log_name,
+    registry_unqualified_name,
+)
+from inspect_ai._util.timeouts import Timeout, timeout
 from inspect_ai._view.notify import view_notify_eval
 from inspect_ai.dataset import Dataset, Sample
 from inspect_ai.log import (
@@ -71,9 +75,9 @@ from inspect_ai.scorer._scorer import unique_scorer_name
 from inspect_ai.solver import Generate, Plan, TaskState
 from inspect_ai.solver._chain import Chain, unroll
 from inspect_ai.solver._fork import set_task_generate
+from inspect_ai.solver._limit import SampleLimitExceededError
 from inspect_ai.solver._solver import Solver
 from inspect_ai.solver._task_state import sample_state, set_sample_state, state_jsonable
-from inspect_ai.util._limit import SampleLimitExceededError
 from inspect_ai.util._sandbox.context import sandbox_connections
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
 from inspect_ai.util._subtask import init_subtask
@@ -398,7 +402,13 @@ async def task_run(options: TaskRunOptions) -> EvalLog:
         view_notify_eval(logger.location)
 
         try:
-            await send_telemetry("eval_log", eval_log_json_str(eval_log))
+            if (
+                await send_telemetry("eval_log_location", eval_log.location)
+                == "not_handled"
+            ):
+                # Converting the eval log to JSON is expensive. Only do so if
+                # eval_log_location was not handled.
+                await send_telemetry("eval_log", eval_log_json_str(eval_log))
         except Exception as ex:
             py_logger.warning(
                 f"Error occurred sending telemetry: {exception_message(ex)}"
@@ -646,26 +656,21 @@ async def task_run_sample(
                     )
 
                     # capture most recent state for scoring
-                    state = sample_state() or state
+                    state = ex.state or sample_state() or state
                     state.completed = True
 
                 except BaseException as ex:
                     error = handle_error(ex)
 
-                # set timeout for scoring. if the original timeout was never hit
-                # then just create a new timeout_cm targeting the original
-                # timeout time. if the original timeout was hit we still want
-                # to provide an opportunity for scoring, but we don't necessarily
+                # set timeout for scoring. if the original timeout was hit we still
+                # want to provide opportunity for scoring, but we don't necessarily
                 # want to wait the full timeout again (especially in the case where
                 # the cause of the timeout is a hung container and scoring requires
                 # interacting with the container). as a middle ground we use half
                 # of the original timeout value for scoring.
                 if isinstance(timeout_cm, Timeout):
-                    if not timeout_cm.expired():
-                        timeout_cm = timeout_at(timeout_cm.when())
-                    else:
-                        assert time_limit
-                        timeout_cm = timeout(time_limit / 2)
+                    assert time_limit
+                    timeout_cm = timeout(time_limit / 2)
 
                 # turn off sample limits
                 set_active_sample_token_limit(None)
@@ -690,6 +695,7 @@ async def task_run_sample(
                                         sample_score = SampleScore(
                                             score=score_result,
                                             sample_id=sample.id,
+                                            scorer=registry_unqualified_name(scorer),
                                         )
                                         transcript()._event(
                                             ScoreEvent(
@@ -734,7 +740,7 @@ async def task_run_sample(
                     error = handle_error(ex)
 
         # handle sandboxenv init errors
-        except BaseException as ex:
+        except Exception as ex:
             error = handle_error(ex)
 
         # complete the sample

inspect_ai/_util/hooks.py

@@ -17,19 +17,29 @@ from .error import PrerequisiteError
 #
 # Telemetry can be optionally enabled by setting an INSPECT_TELEMETRY
 # environment variable that points to a function in a package which
-# conforms to the TelemetrySend signature below.
+# conforms to the TelemetrySend signature below. A return value of True
+# indicates that the telemetry event was handled.
 
-# There are currently two types of telemetry sent:
-#    - model_usage (type ModelUsage)
-#    - eval_log    (type EvalLog)
+# There are currently three types of telemetry sent:
+#    - model_usage       (JSON string of the model usage)
+#    - eval_log_location (file path or URL string of the eval log)
+#    - eval_log          (JSON string of the eval log)
+#                        [only sent if eval_log_location unhandled]
+# The eval_log_location type is preferred over eval_log as it means we can take
+# advantage of the .eval format and avoid loading the whole log into memory.
 
-TelemetrySend = Callable[[str, str], Awaitable[None]]
+TelemetrySend = Callable[[str, str], Awaitable[bool]]
 
 
-async def send_telemetry(type: Literal["model_usage", "eval_log"], json: str) -> None:
+async def send_telemetry(
+    type: Literal["model_usage", "eval_log", "eval_log_location"], json: str
+) -> Literal["handled", "not_handled", "no_subscribers"]:
     global _send_telemetry
     if _send_telemetry:
-        await _send_telemetry(type, json)
+        if await _send_telemetry(type, json):
+            return "handled"
+        return "not_handled"
+    return "no_subscribers"
 
 
 _send_telemetry: TelemetrySend | None = None