inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_cli/cache.py

@@ -42,7 +42,10 @@ def _print_table(title: str, paths: list[tuple[str, int]]) -> None:
 
 @click.group("cache")
 def cache_command() -> None:
-    """Manage the inspect cache."""
+    """Manage the inspect model output cache.
+
+    Learn more about model output caching at https://inspect.ai-safety-institute.org.uk/caching.html.
+    """
     return None
 
 
@@ -62,11 +65,9 @@ def cache_command() -> None:
     type=str,
     help="Clear the cache for a specific model (e.g. --model=openai/gpt-4). Can be passed multiple times.",
 )
-def clear(
-    all: bool, model: tuple[str, ...], log_level: str, log_level_transcript: str
-) -> None:
+def clear(all: bool, model: tuple[str, ...], log_level: str) -> None:
     """Clear all cache files. Requires either --all or --model flags."""
-    init_logger(log_level, log_level_transcript)
+    init_logger(log_level)
 
     if model:
         _print_table(
@@ -119,14 +120,14 @@ def list_caches(pruneable: bool) -> None:
     type=str,
     help="Only prune a specific model (e.g. --model=openai/gpt-4). Can be passed multiple times.",
 )
-def prune(log_level: str, log_level_transcript: str, model: tuple[str, ...]) -> None:
+def prune(log_level: str, model: tuple[str, ...]) -> None:
     """Prune all expired cache entries
 
     Over time the cache directory can grow, but many cache entries will be
     expired. This command will remove all expired cache entries for ease of
     maintenance.
     """
-    init_logger(log_level, log_level_transcript)
+    init_logger(log_level)
 
     expired_cache_entries = cache_list_expired(list(model))
 

inspect_ai/_cli/common.py

@@ -9,14 +9,12 @@ from inspect_ai._util.constants import (
     ALL_LOG_LEVELS,
     DEFAULT_DISPLAY,
     DEFAULT_LOG_LEVEL,
-    DEFAULT_LOG_LEVEL_TRANSCRIPT,
 )
 from inspect_ai.util._display import init_display_type
 
 
 class CommonOptions(TypedDict):
     log_level: str
-    log_level_transcript: str
     log_dir: str
     display: Literal["full", "conversation", "rich", "plain", "none"]
     no_ansi: bool | None
@@ -36,16 +34,6 @@ def log_level_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         envvar="INSPECT_LOG_LEVEL",
         help=f"Set the log level (defaults to '{DEFAULT_LOG_LEVEL}')",
     )
-    @click.option(
-        "--log-level-transcript",
-        type=click.Choice(
-            [level.lower() for level in ALL_LOG_LEVELS],
-            case_sensitive=False,
-        ),
-        default=DEFAULT_LOG_LEVEL_TRANSCRIPT,
-        envvar="INSPECT_LOG_LEVEL_TRANSCRIPT",
-        help=f"Set the log level of the transcript (defaults to '{DEFAULT_LOG_LEVEL_TRANSCRIPT}')",
-    )
     @functools.wraps(func)
     def wrapper(*args: Any, **kwargs: Any) -> click.Context:
         return cast(click.Context, func(*args, **kwargs))

inspect_ai/_cli/eval.py

@@ -7,7 +7,9 @@ from typing_extensions import Unpack
 from inspect_ai import Epochs, eval, eval_retry
 from inspect_ai._eval.evalset import eval_set
 from inspect_ai._util.constants import (
+    ALL_LOG_LEVELS,
     DEFAULT_EPOCHS,
+    DEFAULT_LOG_LEVEL_TRANSCRIPT,
     DEFAULT_MAX_CONNECTIONS,
     DEFAULT_MAX_RETRIES,
 )
@@ -399,6 +401,16 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         envvar=["INSPECT_LOG_FORMAT", "INSPECT_EVAL_LOG_FORMAT"],
         help="Format for writing log files.",
     )
+    @click.option(
+        "--log-level-transcript",
+        type=click.Choice(
+            [level.lower() for level in ALL_LOG_LEVELS],
+            case_sensitive=False,
+        ),
+        default=DEFAULT_LOG_LEVEL_TRANSCRIPT,
+        envvar="INSPECT_LOG_LEVEL_TRANSCRIPT",
+        help=f"Set the log level of the transcript (defaults to '{DEFAULT_LOG_LEVEL_TRANSCRIPT}')",
+    )
     @common_options
     @functools.wraps(func)
     def wrapper(*args: Any, **kwargs: Any) -> click.Context:
@@ -468,6 +480,7 @@ def eval_command(
     no_score: bool | None,
     no_score_display: bool | None,
     log_format: Literal["eval", "json"] | None,
+    log_level_transcript: str,
     **common: Unpack[CommonOptions],
 ) -> None:
     """Evaluate tasks."""
@@ -482,7 +495,7 @@ def eval_command(
         tasks=tasks,
         solver=solver,
         log_level=common["log_level"],
-        log_level_transcript=common["log_level_transcript"],
+        log_level_transcript=log_level_transcript,
         log_dir=common["log_dir"],
         log_format=log_format,
         model=model,
@@ -630,9 +643,13 @@ def eval_set_command(
     bundle_dir: str | None,
     bundle_overwrite: bool | None,
     log_format: Literal["eval", "json"] | None,
+    log_level_transcript: str,
     **common: Unpack[CommonOptions],
 ) -> int:
-    """Evaluate a set of tasks."""
+    """Evaluate a set of tasks with retries.
+
+    Learn more about eval sets at https://inspect.ai-safety-institute.org.uk/eval-sets.html.
+    """
     # read config
     config = config_from_locals(dict(locals()))
 
@@ -644,7 +661,7 @@ def eval_set_command(
         tasks=tasks,
         solver=solver,
         log_level=common["log_level"],
-        log_level_transcript=common["log_level_transcript"],
+        log_level_transcript=log_level_transcript,
         log_dir=common["log_dir"],
         log_format=log_format,
         model=model,
@@ -967,6 +984,16 @@ def parse_comma_separated(value: str | None) -> list[str] | None:
     "--max-retries", type=int, help=MAX_RETRIES_HELP, envvar="INSPECT_EVAL_MAX_RETRIES"
 )
 @click.option("--timeout", type=int, help=TIMEOUT_HELP, envvar="INSPECT_EVAL_TIMEOUT")
+@click.option(
+    "--log-level-transcript",
+    type=click.Choice(
+        [level.lower() for level in ALL_LOG_LEVELS],
+        case_sensitive=False,
+    ),
+    default=DEFAULT_LOG_LEVEL_TRANSCRIPT,
+    envvar="INSPECT_LOG_LEVEL_TRANSCRIPT",
+    help=f"Set the log level of the transcript (defaults to '{DEFAULT_LOG_LEVEL_TRANSCRIPT}')",
+)
 @common_options
 def eval_retry_command(
     log_files: tuple[str],
@@ -986,6 +1013,7 @@ def eval_retry_command(
     max_connections: int | None,
     max_retries: int | None,
     timeout: int | None,
+    log_level_transcript: str,
     **common: Unpack[CommonOptions],
 ) -> None:
     """Retry failed evaluation(s)"""
@@ -1014,7 +1042,7 @@ def eval_retry_command(
     eval_retry(
         retry_log_files,
         log_level=common["log_level"],
-        log_level_transcript=common["log_level_transcript"],
+        log_level_transcript=log_level_transcript,
         log_dir=common["log_dir"],
         max_samples=max_samples,
         max_tasks=max_tasks,

inspect_ai/_cli/info.py

@@ -25,6 +25,7 @@ def info_command() -> None:
     help="Output version and path info as JSON",
 )
 def version(json: bool) -> None:
+    """Output version and path info."""
     if json:
         print(dumps(dict(version=__version__, path=PKG_PATH.as_posix()), indent=2))
     else:

inspect_ai/_cli/list.py

@@ -14,7 +14,7 @@ from inspect_ai._eval.task import TaskInfo
 
 @click.group("list")
 def list_command() -> None:
-    """List tasks or eval logs."""
+    """List tasks on the filesystem."""
     return None
 
 

inspect_ai/_cli/log.py

@@ -29,6 +29,8 @@ def log_command() -> None:
     The default format is 'eval'. You can change this by setting the INSPECT_LOG_FORMAT environment variable or using the --log-format command line option.
 
     The 'log' commands enable you to read Inspect logs uniformly as JSON no matter their physical storage format, and also enable you to read only the headers (everything but the samples) from log files, which is useful for very large logs.
+
+    Learn more about managing log files at https://inspect.ai-safety-institute.org.uk/eval-logs.html.
     """
     return None
 

inspect_ai/_cli/main.py

@@ -53,7 +53,7 @@ inspect.add_command(trace_command)
 def main() -> None:
     set_exception_hook()
     init_dotenv()
-    inspect(auto_envvar_prefix="INSPECT")
+    inspect(auto_envvar_prefix="INSPECT")  # pylint: disable=no-value-for-parameter
 
 
 if __name__ == "__main__":

inspect_ai/_cli/sandbox.py

@@ -7,7 +7,10 @@ from inspect_ai.util._sandbox.registry import registry_find_sandboxenv
 
 @click.group("sandbox")
 def sandbox_command() -> None:
-    """Manage Sandbox Environments."""
+    """Manage Sandbox Environments.
+
+    Learn more about sandboxing at https://inspect.ai-safety-institute.org.uk/sandboxing.html.
+    """
     return None
 
 

inspect_ai/_cli/score.py

@@ -2,33 +2,61 @@ import asyncio
 import os
 
 import click
+import rich
+from rich.panel import Panel
+from rich.prompt import Prompt
+from rich.table import Table
 from typing_extensions import Unpack
 
+from inspect_ai._cli.util import parse_cli_config
 from inspect_ai._display import display
+from inspect_ai._display.core.rich import rich_theme
 from inspect_ai._eval.context import init_eval_context, init_task_context
-from inspect_ai._eval.loader import load_tasks
-from inspect_ai._eval.score import task_score
-from inspect_ai._util.constants import SCORED_SUFFIX
+from inspect_ai._eval.score import ScoreAction, task_score
+from inspect_ai._util.file import basename, dirname, exists
+from inspect_ai.log._log import EvalLog
 from inspect_ai.log._recorders import create_recorder_for_location
 from inspect_ai.model import get_model
 
 from .common import CommonOptions, common_options, process_common_options
 
+SCORES_PER_ROW = 4
+
 
 @click.command("score")
-@click.argument("task", type=str)
 @click.argument("log-file", type=str, required=True)
 @click.option(
-    "--no-overwrite",
+    "--scorer",
+    type=str,
+    envvar="INSPECT_SCORE_SCORER",
+    help="Scorer to use for scoring",
+)
+@click.option(
+    "-S",
+    multiple=True,
+    type=str,
+    envvar="INSPECT_SCORE_SCORER_ARGS",
+    help="One or more scorer arguments (e.g. -S arg=value)",
+)
+@click.option(
+    "--action",
+    type=click.Choice(["append", "overwrite"]),
+    envvar="INSPECT_SCORE_SCORER_ACTION",
+    help="Whether to append or overwrite the existing scores.",
+)
+@click.option(
+    "--overwrite",
     type=bool,
     is_flag=True,
-    help="Do not overwrite unscored log_files with the scored version (instead write a new file w/ '-scored' appended)",
+    help="Overwrite log file with the scored version",
 )
 @common_options
 def score_command(
-    task: str,
     log_file: str,
-    no_overwrite: bool | None,
+    overwrite: bool | None,
+    scorer: str | None,
+    s: tuple[str] | None,
+    action: ScoreAction | None,
     **common: Unpack[CommonOptions],
 ) -> None:
     """Score a previous evaluation run."""
@@ -38,31 +66,43 @@ def score_command(
     # score
     asyncio.run(
         score(
-            task,
-            common["log_dir"],
-            log_file,
-            False if no_overwrite else True,
-            common["log_level"],
-            common["log_level_transcript"],
+            log_dir=common["log_dir"],
+            log_file=log_file,
+            scorer=scorer,
+            s=s,
+            overwrite=False if overwrite is None else overwrite,
+            action=action,
+            log_level=common["log_level"],
         )
     )
 
 
 async def score(
-    task: str,
     log_dir: str,
     log_file: str,
+    scorer: str | None,
+    s: tuple[str] | None,
     overwrite: bool,
+    action: ScoreAction | None,
     log_level: str | None,
-    log_level_transcript: str | None,
+    output_file: str | None = None,
 ) -> None:
     # init eval context
-    init_eval_context(log_level, log_level_transcript)
+    init_eval_context(log_level, None)
+    scorer_args = parse_cli_config(args=s, config=None)
 
     # read the eval log
     recorder = create_recorder_for_location(log_file, log_dir)
     eval_log = await recorder.read_log(log_file)
 
+    # resolve the target output file (prompts user)
+    output_file = resolve_output_file(
+        log_file, output_file=output_file, overwrite=overwrite
+    )
+
+    # resolve action
+    action = resolve_action(eval_log, action)
+
     # check that there are samples therein
     if eval_log.samples is None or len(eval_log.samples) == 0:
         raise ValueError(f"{log_file} does not include samples to score")
@@ -77,23 +117,132 @@ async def score(
     # initialize active model
     init_task_context(model)
 
-    # instantiate the task so we can get its scorer and metrics
-    score_task = load_tasks([task], model)[0]
-
     # re-score the task
-    eval_log = await task_score(score_task, eval_log)
+    eval_log = await task_score(
+        log=eval_log, scorer=scorer, scorer_args=scorer_args, action=action
+    )
 
-    # re-write the log (w/ a -score suffix if requested)
-    _, ext = os.path.splitext(log_file)
-    scored = f"{SCORED_SUFFIX}{ext}"
-    if not overwrite and not log_file.endswith(scored):
-        log_file = log_file.removesuffix(ext) + scored
-    await recorder.write_log(log_file, eval_log)
+    # re-write the log
+    await recorder.write_log(output_file, eval_log)
 
     # print results
-    display().print(f"\n{eval_log.eval.task}")
+    print_results(output_file, eval_log)
+
+
+def print_results(output_file: str, eval_log: EvalLog) -> None:
+    # the theme
+    theme = rich_theme()
+
+    # Create results panel
+    grid = Table.grid(expand=True)
+    grid.add_column()
+    grid.add_row("")
+
     if eval_log.results:
-        for score in eval_log.results.scores:
-            for name, metric in score.metrics.items():
-                display().print(f"{name}: {metric.value}")
-    display().print(f"log: {log_file}\n")
+        # Process scores in groups
+        for i in range(0, len(eval_log.results.scores), SCORES_PER_ROW):
+            # Create a grid for this row of scores
+            score_row = Table.grid(
+                expand=False,
+                padding=(0, 2, 0, 0),
+            )
+
+            # Add columns for each score in this row
+            for _ in range(SCORES_PER_ROW):
+                score_row.add_column()
+
+            # Create individual score tables and add them to the row
+            score_tables: list[Table | str] = []
+            for score in eval_log.results.scores[i : i + SCORES_PER_ROW]:
+                table = Table(
+                    show_header=False, show_lines=False, box=None, show_edge=False
+                )
+                table.add_column()
+                table.add_column()
+
+                # Add score name and metrics
+                table.add_row(f"[bold]{score.name}[/bold]")
+                for name, metric in score.metrics.items():
+                    table.add_row(f"{name}", f"{metric.value:.3f}")
+
+                score_tables.append(table)
+
+            # Fill remaining slots with empty tables if needed
+            while len(score_tables) < SCORES_PER_ROW:
+                score_tables.append("")
+
+            # Add the score tables to this row
+            score_row.add_row(*score_tables)
+
+            # Add this row of scores to the main grid
+            grid.add_row(score_row)
+
+    grid.add_row("")
+    grid.add_row(f" Log: [{theme.link}]{output_file}[/{theme.link}]")
+
+    p = Panel(
+        title=f"[bold][{theme.meta}]Results for {eval_log.eval.task}[/bold][/{theme.meta}]",
+        title_align="left",
+        renderable=grid,
+    )
+
+    # Print the results panel
+    display().print("")
+    console = rich.get_console()
+    console.print(p)
+
+
+def resolve_output_file(log_file: str, output_file: str | None, overwrite: bool) -> str:
+    # resolve the output file (we may overwrite, use the passed file name, or suggest a new name)
+    if output_file is None:
+        if overwrite:
+            # explicitly asked to overwrite
+            return log_file
+        else:
+            if exists(log_file):
+                # Ask if we should overwrite
+                file_action = Prompt.ask(
+                    "Overwrite existing log file or create new log file?",
+                    choices=["overwrite", "create", "o", "c"],
+                    default="create",
+                )
+                if file_action in ["overwrite", "o"]:
+                    return log_file
+                else:
+                    file_name = basename(log_file)
+                    base_dir = dirname(log_file)
+                    _, ext = os.path.splitext(file_name)
+
+                    count = 0
+
+                    def filename() -> str:
+                        if count > 0:
+                            return f"{file_name.removesuffix(ext)}-scored-{count}{ext}"
+                        else:
+                            return f"{file_name.removesuffix(ext)}-scored{ext}"
+
+                    while exists(f"{os.path.join(base_dir, filename())}"):
+                        count = count + 1
+
+                    suggested_file = filename()
+                    user_file = Prompt.ask("Output file name?", default=suggested_file)
+                    return os.path.join(base_dir, user_file)
+            else:
+                return log_file
+    else:
+        return output_file
+
+
+def resolve_action(eval_log: EvalLog, action: ScoreAction | None) -> ScoreAction:
+    if action is not None:
+        return action
+
+    if eval_log.results is not None and len(eval_log.results.scores) > 0:
+        user_action = Prompt.ask(
+            "Overwrite existing scores or append as additional scores?",
+            choices=["overwrite", "append", "o", "a"],
+            default="append",
+        )
+        return "overwrite" if user_action in ["ovewrite", "o"] else "append"
+    else:
+        return "overwrite"

inspect_ai/_cli/trace.py

@@ -26,6 +26,8 @@ def trace_command() -> None:
     """List and read execution traces.
 
     Inspect includes a TRACE log-level which is right below the HTTP and INFO log levels (so not written to the console by default). However, TRACE logs are always recorded to a separate file, and the last 10 TRACE logs are preserved. The 'trace' command provides ways to list and read these traces.
+
+    Learn more about execution traces at https://inspect.ai-safety-institute.org.uk/tracing.html.
     """
     return None
 
@@ -109,11 +111,13 @@ def anomolies_command(trace_file: str | None, filter: str | None, all: bool) ->
     canceled_actions: dict[str, ActionTraceRecord] = {}
     error_actions: dict[str, ActionTraceRecord] = {}
     timeout_actions: dict[str, ActionTraceRecord] = {}
+    start_trace: ActionTraceRecord | None = None
 
     def action_started(trace: ActionTraceRecord) -> None:
         running_actions[trace.trace_id] = trace
 
     def action_completed(trace: ActionTraceRecord) -> ActionTraceRecord:
+        nonlocal start_trace
         start_trace = running_actions.get(trace.trace_id)
         if start_trace:
             del running_actions[trace.trace_id]
@@ -122,14 +126,20 @@ def anomolies_command(trace_file: str | None, filter: str | None, all: bool) ->
             raise RuntimeError(f"Expected {trace.trace_id} in action dictionary.")
 
     def action_failed(trace: ActionTraceRecord) -> None:
+        nonlocal start_trace
         if all:
+            assert start_trace
             error_actions[start_trace.trace_id] = trace
 
     def action_canceled(trace: ActionTraceRecord) -> None:
+        nonlocal start_trace
+        assert start_trace
         canceled_actions[start_trace.trace_id] = trace
 
     def action_timeout(trace: ActionTraceRecord) -> None:
+        nonlocal start_trace
         if all:
+            assert start_trace
             timeout_actions[start_trace.trace_id] = trace
 
     for trace in traces:

inspect_ai/_cli/view.py

@@ -39,7 +39,10 @@ def start_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
 @common_options
 @click.pass_context
 def view_command(ctx: click.Context, **kwargs: Unpack[CommonOptions]) -> None:
-    """View command group."""
+    """Inspect log viewer.
+
+    Learn more about using the log viewer at https://inspect.ai-safety-institute.org.uk/log-viewer.html.
+    """
     if ctx.invoked_subcommand is None:
         ctx.invoke(start, **kwargs)
     else:
@@ -78,7 +81,6 @@ def start(
         port=port,
         authorization=authorization,
         log_level=common["log_level"],
-        log_level_transcript=common["log_level_transcript"],
     )
 
 

inspect_ai/_display/core/active.py

@@ -10,6 +10,8 @@ from ..rich.display import RichDisplay
 from ..textual.display import TextualDisplay
 from .display import Display, TaskScreen
 
+_active_display: Display | None = None
+
 
 def display() -> Display:
     global _active_display
@@ -28,9 +30,6 @@ def display() -> Display:
     return _active_display
 
 
-_active_display: Display | None = None
-
-
 def task_screen() -> TaskScreen:
     screen = _active_task_screen.get(None)
     if screen is None:

inspect_ai/_display/core/config.py

@@ -1,4 +1,5 @@
 from inspect_ai._util.registry import is_registry_dict
+from inspect_ai.log._log import eval_config_defaults
 
 from .display import TaskProfile
 
@@ -13,7 +14,12 @@ def task_config(
         value = task_args[key]
         if is_registry_dict(value):
             task_args[key] = value["name"]
-    config = dict(profile.eval_config.model_dump(exclude_none=True)) | task_args
+    # get eval_config overrides
+    eval_config = dict(profile.eval_config.model_dump(exclude_none=True))
+    for name, default_value in eval_config_defaults().items():
+        if eval_config.get(name, None) == default_value:
+            del eval_config[name]
+    config = eval_config | task_args
     if generate_config:
         config = dict(profile.generate_config.model_dump(exclude_none=True)) | config
     if profile.tags:

inspect_ai/_display/textual/widgets/samples.py

@@ -347,7 +347,7 @@ class SampleLimits(Widget):
 class SandboxesView(Vertical):
     DEFAULT_CSS = """
     SandboxesView {
-        padding: 1 0 1 0;
+        padding: 1 0 0 0;
         background: transparent;
         height: auto;
     }
@@ -358,6 +358,7 @@ class SandboxesView(Vertical):
         background: transparent;
     }
     .clipboard-message {
+        height: auto;
         margin-top: 1;
     }
     """
@@ -372,7 +373,6 @@ class SandboxesView(Vertical):
     async def sync_sample(self, sample: ActiveSample) -> None:
         if len(sample.sandboxes) > 0:
             multiple_sandboxes = len(sample.sandboxes) > 1
-            self.display = True
             sandboxes_caption = cast(Static, self.query_one("#sandboxes-caption"))
             sandboxes_caption.update(
                 f"[bold]sandbox container{'s' if multiple_sandboxes else ''}:[/bold]"
@@ -395,6 +395,7 @@ class SandboxesView(Vertical):
                     markup=True,
                 )
             )
+            self.display = True
         else:
             self.display = False
 
@@ -473,7 +474,7 @@ class SampleToolbar(Horizontal):
                     else None
                 )
                 if isinstance(last_event, ToolEvent):
-                    last_event.cancel()
+                    last_event._cancel()
             elif event.button.id == self.CANCEL_SCORE_OUTPUT:
                 self.sample.interrupt("score")
             elif event.button.id == self.CANCEL_RAISE_ERROR:

inspect_ai/_display/textual/widgets/sandbox.py

@@ -9,6 +9,12 @@ from .port_mappings import PortMappingsView
 
 class SandboxView(Vertical):
     DEFAULT_CSS = """
+    SandboxView {
+        height: auto;
+    }
+    SandboxView * {
+        height: auto;
+    }
     .indent {
         width: 2;
     }