inspect-ai 0.3.91__py3-none-any.whl → 0.3.93__py3-none-any.whl

inspect_ai/_cli/eval.py

@@ -43,6 +43,9 @@ MAX_SANDBOXES_HELP = "Maximum number of sandboxes (per-provider) to run in paral
 NO_SANDBOX_CLEANUP_HELP = "Do not cleanup sandbox environments after task completes"
 FAIL_ON_ERROR_HELP = "Threshold of sample errors to tolerage (by default, evals fail when any error occurs). Value between 0 to 1 to set a proportion; value greater than 1 to set a count."
 NO_LOG_SAMPLES_HELP = "Do not include samples in the log file."
+NO_LOG_REALTIME_HELP = (
+    "Do not log events in realtime (affects live viewing of samples in inspect view)"
+)
 NO_FAIL_ON_ERROR_HELP = "Do not fail the eval if errors occur within samples (instead, continue running other samples)"
 RETRY_ON_ERROR_HELP = "Retry samples if they encounter errors (by default, no retries occur). Specify --retry-on-error to retry a single time, or specify e.g. `--retry-on-error=3` to retry multiple times."
 LOG_IMAGES_HELP = (
@@ -281,6 +284,13 @@ def eval_options(func: Callable[..., Any]) -> Callable[..., click.Context]:
         help=NO_LOG_SAMPLES_HELP,
         envvar="INSPECT_EVAL_NO_LOG_SAMPLES",
     )
+    @click.option(
+        "--no-log-realtime",
+        type=bool,
+        is_flag=True,
+        help=NO_LOG_REALTIME_HELP,
+        envvar="INSPECT_EVAL_NO_LOG_REALTIME",
+    )
     @click.option(
         "--log-images/--no-log-images",
         type=bool,
@@ -544,6 +554,7 @@ def eval_command(
     no_fail_on_error: bool | None,
     retry_on_error: int | None,
     no_log_samples: bool | None,
+    no_log_realtime: bool | None,
     log_images: bool | None,
     log_buffer: int | None,
     log_shared: int | None,
@@ -600,6 +611,7 @@ def eval_command(
         retry_on_error=retry_on_error,
         debug_errors=common["debug_errors"],
         no_log_samples=no_log_samples,
+        no_log_realtime=no_log_realtime,
         log_images=log_images,
         log_buffer=log_buffer,
         log_shared=log_shared,
@@ -718,6 +730,7 @@ def eval_set_command(
     no_fail_on_error: bool | None,
     retry_on_error: int | None,
     no_log_samples: bool | None,
+    no_log_realtime: bool | None,
     log_images: bool | None,
     log_buffer: int | None,
     log_shared: int | None,
@@ -779,6 +792,7 @@ def eval_set_command(
         retry_on_error=retry_on_error,
         debug_errors=common["debug_errors"],
         no_log_samples=no_log_samples,
+        no_log_realtime=no_log_realtime,
         log_images=log_images,
         log_buffer=log_buffer,
         log_shared=log_shared,
@@ -837,6 +851,7 @@ def eval_exec(
     retry_on_error: int | None,
     debug_errors: bool | None,
     no_log_samples: bool | None,
+    no_log_realtime: bool | None,
     log_images: bool | None,
     log_buffer: int | None,
     log_shared: int | None,
@@ -889,6 +904,7 @@ def eval_exec(
     # resolve negating options
     sandbox_cleanup = False if no_sandbox_cleanup else None
     log_samples = False if no_log_samples else None
+    log_realtime = False if no_log_realtime else None
     log_images = False if log_images is False else None
     trace = True if trace else None
     score = False if no_score else True
@@ -929,6 +945,7 @@ def eval_exec(
             max_subprocesses=max_subprocesses,
             max_sandboxes=max_sandboxes,
             log_samples=log_samples,
+            log_realtime=log_realtime,
             log_images=log_images,
             log_buffer=log_buffer,
             log_shared=log_shared,
@@ -1069,6 +1086,13 @@ def parse_comma_separated(value: str | None) -> list[str] | None:
     help=NO_LOG_SAMPLES_HELP,
     envvar="INSPECT_EVAL_LOG_SAMPLES",
 )
+@click.option(
+    "--no-log-realtime",
+    type=bool,
+    is_flag=True,
+    help=NO_LOG_REALTIME_HELP,
+    envvar="INSPECT_EVAL_LOG_REALTIME",
+)
 @click.option(
     "--log-images/--no-log-images",
     type=bool,
@@ -1136,6 +1160,7 @@ def eval_retry_command(
     no_fail_on_error: bool | None,
     retry_on_error: int | None,
     no_log_samples: bool | None,
+    no_log_realtime: bool | None,
     log_images: bool | None,
     log_buffer: int | None,
     log_shared: int | None,
@@ -1154,6 +1179,7 @@ def eval_retry_command(
     # resolve negating options
     sandbox_cleanup = False if no_sandbox_cleanup else None
     log_samples = False if no_log_samples else None
+    log_realtime = False if no_log_realtime else None
     log_images = False if log_images is False else None
     score = False if no_score else True
     score_display = False if no_score_display else None
@@ -1164,6 +1190,10 @@ def eval_retry_command(
     elif fail_on_error == 0.0:
         fail_on_error = True
 
+    # resolve retry on error
+    if retry_on_error == 0:
+        retry_on_error = None
+
     # resolve log file
     retry_log_files = [
         log_file_info(filesystem(log_file).info(log_file)) for log_file in log_files
@@ -1185,6 +1215,7 @@ def eval_retry_command(
         retry_on_error=retry_on_error,
         debug_errors=common["debug_errors"],
         log_samples=log_samples,
+        log_realtime=log_realtime,
         log_images=log_images,
         log_buffer=log_buffer,
         log_shared=log_shared,

inspect_ai/_eval/eval.py

@@ -101,6 +101,7 @@ def eval(
     max_subprocesses: int | None = None,
     max_sandboxes: int | None = None,
     log_samples: bool | None = None,
+    log_realtime: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
     log_shared: bool | int | None = None,
@@ -145,7 +146,7 @@ def eval(
             to "eval", the native high-performance format).
         limit: Limit evaluated samples
             (defaults to all samples).
-        sample_id: Evaluate specific sample(s) from the dataset.
+        sample_id: Evaluate specific sample(s) from the dataset. Use plain ids or preface with task names as required to disambiguate ids across tasks (e.g. `popularity:10`).
         epochs: Epochs to repeat samples for and optional score
             reducer function(s) used to combine sample scores (defaults to "mean")
         fail_on_error: `True` to fail on first sample error
@@ -171,6 +172,7 @@ def eval(
         max_sandboxes: Maximum number of sandboxes (per-provider)
             to run in parallel.
         log_samples: Log detailed samples and scores (defaults to True)
+        log_realtime: Log events in realtime (enables live viewing of samples in inspect view). Defaults to True.
         log_images: Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
         log_buffer: Number of samples to buffer before writing log file.
@@ -228,6 +230,7 @@ def eval(
                 max_subprocesses=max_subprocesses,
                 max_sandboxes=max_sandboxes,
                 log_samples=log_samples,
+                log_realtime=log_realtime,
                 log_images=log_images,
                 log_buffer=log_buffer,
                 log_shared=log_shared,
@@ -281,6 +284,7 @@ async def eval_async(
     max_subprocesses: int | None = None,
     max_sandboxes: int | None = None,
     log_samples: bool | None = None,
+    log_realtime: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
     log_shared: bool | int | None = None,
@@ -314,7 +318,7 @@ async def eval_async(
         log_dir: Output path for logging results (defaults to file log in ./logs directory).
         log_format: Format for writing log files (defaults to "eval", the native high-performance format).
         limit: Limit evaluated samples (defaults to all samples).
-        sample_id: Evaluate specific sample(s) from the dataset.
+        sample_id: Evaluate specific sample(s) from the dataset. Use plain ids or preface with task names as required to disambiguate ids across tasks (e.g. `popularity:10`).
         epochs: Epochs to repeat samples for and optional score
             reducer function(s) used to combine sample scores (defaults to "mean")
         fail_on_error: `True` to fail on first sample error
@@ -335,6 +339,7 @@ async def eval_async(
         max_subprocesses: Maximum number of subprocesses to run in parallel (default is os.cpu_count())
         max_sandboxes: Maximum number of sandboxes (per-provider) to run in parallel.
         log_samples: Log detailed samples and scores (defaults to True)
+        log_realtime: Log events in realtime (enables live viewing of samples in inspect view). Defaults to True.
         log_images: Log base64 encoded version of images, even if specified as a filename or URL (defaults to False)
         log_buffer: Number of samples to buffer before writing log file.
            If not specified, an appropriate default for the format and filesystem is
@@ -473,6 +478,7 @@ async def eval_async(
             max_sandboxes=max_sandboxes,
             sandbox_cleanup=sandbox_cleanup,
             log_samples=log_samples,
+            log_realtime=log_realtime,
             log_images=log_images,
             log_buffer=log_buffer,
             log_shared=log_shared,
@@ -562,6 +568,7 @@ def eval_retry(
     retry_on_error: int | None = None,
     debug_errors: bool | None = None,
     log_samples: bool | None = None,
+    log_realtime: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
     log_shared: bool | int | None = None,
@@ -603,6 +610,7 @@ def eval_retry(
         debug_errors: Raise task errors (rather than logging them)
             so they can be debugged (defaults to False).
         log_samples: Log detailed samples and scores (defaults to True)
+        log_realtime: Log events in realtime (enables live viewing of samples in inspect view). Defaults to True.
         log_images: Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
         log_buffer: Number of samples to buffer before writing log file.
@@ -645,6 +653,7 @@ def eval_retry(
             retry_on_error=retry_on_error,
             debug_errors=debug_errors,
             log_samples=log_samples,
+            log_realtime=log_realtime,
             log_images=log_images,
             log_buffer=log_buffer,
             log_shared=log_shared,
@@ -673,6 +682,7 @@ async def eval_retry_async(
     retry_on_error: int | None = None,
     debug_errors: bool | None = None,
     log_samples: bool | None = None,
+    log_realtime: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
     log_shared: bool | int | None = None,
@@ -707,6 +717,7 @@ async def eval_retry_async(
         debug_errors: Raise task errors (rather than logging them)
            so they can be debugged (defaults to False).
         log_samples: Log detailed samples and scores (defaults to True)
+        log_realtime: Log events in realtime (enables live viewing of samples in inspect view). Defaults to True.
         log_images: Log base64 encoded version of images,
            even if specified as a filename or URL (defaults to False)
         log_buffer: Number of samples to buffer before writing log file.
@@ -817,6 +828,11 @@ async def eval_retry_async(
         log_samples = (
             log_samples if log_samples is not None else eval_log.eval.config.log_samples
         )
+        log_realtime = (
+            log_realtime
+            if log_realtime is not None
+            else eval_log.eval.config.log_realtime
+        )
         log_images = (
             log_images if log_images is not None else eval_log.eval.config.log_images
         )
@@ -875,6 +891,7 @@ async def eval_retry_async(
                 max_subprocesses=max_subprocesses,
                 max_sandboxes=max_sandboxes,
                 log_samples=log_samples,
+                log_realtime=log_realtime,
                 log_images=log_images,
                 log_buffer=log_buffer,
                 log_shared=log_shared,

inspect_ai/_eval/evalset.py

@@ -93,6 +93,7 @@ def eval_set(
     max_subprocesses: int | None = None,
     max_sandboxes: int | None = None,
     log_samples: bool | None = None,
+    log_realtime: bool | None = None,
     log_images: bool | None = None,
     log_buffer: int | None = None,
     log_shared: bool | int | None = None,
@@ -147,7 +148,7 @@ def eval_set(
             log files (defaults to "eval", the native high-performance format).
         limit: Limit evaluated samples
             (defaults to all samples).
-        sample_id: Evaluate specific sample(s) from the dataset.
+        sample_id: Evaluate specific sample(s) from the dataset. Use plain ids or preface with task names as required to disambiguate ids across tasks (e.g. `popularity:10`).
         epochs: Epochs to repeat samples for and optional score
             reducer function(s) used to combine sample scores (defaults to "mean")
         fail_on_error: `True` to fail on first sample error
@@ -173,6 +174,7 @@ def eval_set(
         max_sandboxes: Maximum number of sandboxes (per-provider)
             to run in parallel.
         log_samples: Log detailed samples and scores (defaults to True)
+        log_realtime: Log events in realtime (enables live viewing of samples in inspect view). Defaults to True.
         log_images: Log base64 encoded version of images,
             even if specified as a filename or URL (defaults to False)
         log_buffer: Number of samples to buffer before writing log file.
@@ -229,6 +231,7 @@ def eval_set(
             max_subprocesses=max_subprocesses,
             max_sandboxes=max_sandboxes,
             log_samples=log_samples,
+            log_realtime=log_realtime,
             log_images=log_images,
             log_buffer=log_buffer,
             log_shared=log_shared,

inspect_ai/_eval/run.py

@@ -122,6 +122,11 @@ async def eval_run(
                 task = resolved_task.task
                 task_eval_config = eval_config.model_copy()
 
+                # sample_ids can be specified per task
+                task_eval_config.sample_id = resolve_task_sample_ids(
+                    resolved_task.task.name, task_eval_config.sample_id
+                )
+
                 # resolve the task scorers
                 eval_scorer_specs = (
                     [as_scorer_spec(scorer) for scorer in task.scorer]
@@ -424,6 +429,42 @@ async def run_multiple(tasks: list[TaskRunOptions], parallel: int) -> list[EvalL
         return results
 
 
+def resolve_task_sample_ids(
+    task: str, sample_id: str | int | list[str] | list[int] | list[str | int] | None
+) -> str | int | list[str] | list[int] | list[str | int] | None:
+    def collect_for_task(sample: str | int) -> str | int | None:
+        if isinstance(sample, str):
+            scoped = sample.split(":", maxsplit=1)
+            if len(scoped) > 1:
+                if scoped[0].lower() == task.lower():
+                    return scoped[1]
+                else:
+                    return None
+            else:
+                return sample
+        else:
+            return sample
+
+    if sample_id is not None:
+        if isinstance(sample_id, list):
+            ids: list[int | str] = []
+            for id in sample_id:
+                collect = collect_for_task(id)
+                if collect is not None:
+                    ids.append(collect)
+            return ids
+
+        else:
+            collect = collect_for_task(sample_id)
+            if collect is not None:
+                return collect
+            else:
+                return []
+
+    else:
+        return sample_id
+
+
 async def startup_sandbox_environments(
     eval_sandbox: SandboxEnvironmentSpec | None,
     tasks: list[ResolvedTask],

inspect_ai/_eval/task/generate.py

@@ -4,7 +4,6 @@ from inspect_ai.model import CachePolicy, GenerateConfig, Model
 from inspect_ai.model._cache import epoch
 from inspect_ai.model._call_tools import execute_tools
 from inspect_ai.solver import TaskState
-from inspect_ai.solver._limit import SampleLimitExceededError
 from inspect_ai.tool import ToolFunction
 
 
@@ -18,53 +17,48 @@ async def task_generate(
     # track tool_choice (revert to "auto" after first forced call of a tool)
     tool_choice = state.tool_choice
 
-    try:
-        while True:
-            # If we don't update the epoch here as we go, it's entirely possible
-            # we'd cache the same response for every single epoch, which would
-            # completely defeat the point!
-            epoch.set(state.epoch)
+    while True:
+        # If we don't update the epoch here as we go, it's entirely possible
+        # we'd cache the same response for every single epoch, which would
+        # completely defeat the point!
+        epoch.set(state.epoch)
 
-            # call the model
-            state.output = await model.generate(
-                input=state.messages,
-                tools=state.tools,
-                tool_choice=tool_choice,
-                config=config,
-                cache=cache,
-            )
-
-            # append the assistant message
-            message = state.output.message
-            state.messages.append(message)
-
-            # check for completed
-            if state.completed:
-                return state
+        # call the model
+        state.output = await model.generate(
+            input=state.messages,
+            tools=state.tools,
+            tool_choice=tool_choice,
+            config=config,
+            cache=cache,
+        )
 
-            # resolve tool calls if necessary
-            if tool_calls != "none" and message.tool_calls:
-                # call tools and update messages and output
-                messages, output = await execute_tools(
-                    state.messages, state.tools, config.max_tool_output
-                )
-                state.messages.extend(messages)
-                if output is not None:
-                    state.output = output
+        # append the assistant message
+        message = state.output.message
+        state.messages.append(message)
 
-                # check for completed or only executing a single tool call
-                if state.completed or tool_calls == "single":
-                    return state
+        # check for completed
+        if state.completed:
+            return state
 
-                # if a tool_call was forced set tool_choice to 'auto'
-                # (otherwise it will get forced over and over again)
-                if isinstance(tool_choice, ToolFunction):
-                    tool_choice = "auto"
+        # resolve tool calls if necessary
+        if tool_calls != "none" and message.tool_calls:
+            # call tools and update messages and output
+            messages, output = await execute_tools(
+                state.messages, state.tools, config.max_tool_output
+            )
+            state.messages.extend(messages)
+            if output is not None:
+                state.output = output
 
-            # no tool calls or not resolving tool calls, we are done!
-            else:
+            # check for completed or only executing a single tool call
+            if state.completed or tool_calls == "single":
                 return state
 
-    # propagate current state along with sample limit exceeded
-    except SampleLimitExceededError as ex:
-        raise ex.with_state(state)
+            # if a tool_call was forced set tool_choice to 'auto'
+            # (otherwise it will get forced over and over again)
+            if isinstance(tool_choice, ToolFunction):
+                tool_choice = "auto"
+
+        # no tool calls or not resolving tool calls, we are done!
+        else:
+            return state

inspect_ai/_eval/task/log.py

@@ -30,13 +30,14 @@ from inspect_ai.log._log import (
     EvalLog,
     EvalMetricDefinition,
     EvalSampleReductions,
+    EvalSampleSummary,
     EvalScorer,
     eval_config_defaults,
 )
 from inspect_ai.log._model import model_args_for_log, model_roles_to_model_roles_config
 from inspect_ai.log._recorders import Recorder
 from inspect_ai.log._recorders.buffer import SampleBufferDatabase
-from inspect_ai.log._recorders.types import SampleEvent, SampleSummary
+from inspect_ai.log._recorders.types import SampleEvent
 from inspect_ai.log._transcript import Event
 from inspect_ai.model import (
     GenerateConfig,
@@ -160,13 +161,17 @@ class TaskLogger:
         self.flush_buffer = eval_config.log_buffer or recorder.default_log_buffer()
         self.flush_pending: list[tuple[str | int, int]] = []
 
+        # sample buffer db
+        self._buffer_db: SampleBufferDatabase | None = None
+
     async def init(self) -> None:
         self._location = await self.recorder.log_init(self.eval)
-        self._buffer_db = SampleBufferDatabase(
-            location=self._location,
-            log_images=self.eval.config.log_images is not False,
-            log_shared=self.eval.config.log_shared,
-        )
+        if self.eval.config.log_realtime is not False:
+            self._buffer_db = SampleBufferDatabase(
+                location=self._location,
+                log_images=self.eval.config.log_images is not False,
+                log_shared=self.eval.config.log_shared,
+            )
 
     @property
     def location(self) -> str:
@@ -180,36 +185,26 @@ class TaskLogger:
         await self.recorder.log_start(self.eval, plan)
         await self.recorder.flush(self.eval)
 
-    async def start_sample(self, sample: SampleSummary) -> None:
-        self._buffer_db.start_sample(sample)
+    async def start_sample(self, sample: EvalSampleSummary) -> None:
+        if self._buffer_db is not None:
+            self._buffer_db.start_sample(sample)
 
     def log_sample_event(self, id: str | int, epoch: int, event: Event) -> None:
         # log the sample event
-        self._buffer_db.log_events([SampleEvent(id=id, epoch=epoch, event=event)])
+        if self._buffer_db is not None:
+            self._buffer_db.log_events([SampleEvent(id=id, epoch=epoch, event=event)])
 
     def remove_sample(self, id: str | int, epoch: int) -> None:
-        self._buffer_db.remove_samples([(id, epoch)])
+        if self._buffer_db is not None:
+            self._buffer_db.remove_samples([(id, epoch)])
 
     async def complete_sample(self, sample: EvalSample, *, flush: bool) -> None:
         # log the sample
         await self.recorder.log_sample(self.eval, sample)
 
         # mark complete
-        self._buffer_db.complete_sample(
-            SampleSummary(
-                id=sample.id,
-                epoch=sample.epoch,
-                input=sample.input,
-                target=sample.target,
-                completed=True,
-                scores=sample.scores,
-                error=sample.error.message if sample.error is not None else None,
-                limit=f"{sample.limit.type}" if sample.limit is not None else None,
-                retries=len(sample.error_retries)
-                if sample.error_retries is not None
-                else None,
-            )
-        )
+        if self._buffer_db is not None:
+            self._buffer_db.complete_sample(sample.summary())
 
         # flush if requested
         if flush:
@@ -219,7 +214,8 @@ class TaskLogger:
                 await self.recorder.flush(self.eval)
 
                 # notify the event db it can remove these
-                self._buffer_db.remove_samples(self.flush_pending)
+                if self._buffer_db is not None:
+                    self._buffer_db.remove_samples(self.flush_pending)
 
                 # Clear
                 self.flush_pending.clear()
@@ -229,7 +225,8 @@ class TaskLogger:
             self._samples_completed += 1
 
     def update_metrics(self, metrics: list[TaskDisplayMetric]) -> None:
-        self._buffer_db.update_metrics(metrics)
+        if self._buffer_db is not None:
+            self._buffer_db.update_metrics(metrics)
 
     async def log_finish(
         self,
@@ -245,7 +242,8 @@ class TaskLogger:
         )
 
         # cleanup the events db
-        self._buffer_db.cleanup()
+        if self._buffer_db is not None:
+            self._buffer_db.cleanup()
 
         # return log
         return log

inspect_ai/_eval/task/run.py

@@ -51,8 +51,12 @@ from inspect_ai.log import (
 )
 from inspect_ai.log._condense import condense_sample
 from inspect_ai.log._file import eval_log_json_str
-from inspect_ai.log._log import EvalSampleLimit, EvalSampleReductions, eval_error
-from inspect_ai.log._recorders.types import SampleSummary
+from inspect_ai.log._log import (
+    EvalSampleLimit,
+    EvalSampleReductions,
+    EvalSampleSummary,
+    eval_error,
+)
 from inspect_ai.log._samples import (
     active_sample,
 )
@@ -82,9 +86,9 @@ from inspect_ai.scorer._scorer import unique_scorer_name
 from inspect_ai.solver import Generate, Plan, TaskState
 from inspect_ai.solver._chain import Chain, unroll
 from inspect_ai.solver._fork import set_task_generate
-from inspect_ai.solver._limit import SampleLimitExceededError
 from inspect_ai.solver._solver import Solver
 from inspect_ai.solver._task_state import sample_state, set_sample_state, state_jsonable
+from inspect_ai.util._limit import LimitExceededError
 from inspect_ai.util._sandbox.context import sandbox_connections
 from inspect_ai.util._sandbox.environment import SandboxEnvironmentSpec
 from inspect_ai.util._subtask import init_subtask
@@ -649,17 +653,18 @@ async def task_run_sample(
                     init_sample_working_limit(start_time, working_limit)
 
                     # run sample w/ optional timeout
-                    with timeout_cm:
+                    with timeout_cm, state._token_limit, state._message_limit:
                         # mark started
                         active.started = datetime.now().timestamp()
 
                         if logger is not None:
                             await logger.start_sample(
-                                SampleSummary(
+                                EvalSampleSummary(
                                     id=sample_id,
                                     epoch=state.epoch,
                                     input=sample.input,
                                     target=sample.target,
+                                    metadata=sample.metadata or {},
                                 )
                             )
 
@@ -707,18 +712,9 @@ async def task_run_sample(
                         # handle the cancel exception
                         raise
 
-                except SampleLimitExceededError as ex:
-                    # sample limit event
-                    transcript()._event(
-                        SampleLimitEvent(
-                            type=ex.type,
-                            limit=ex.limit,
-                            message=f"Sample completed: {ex.message}",
-                        )
-                    )
-
+                except LimitExceededError:
                     # capture most recent state for scoring
-                    state = ex.state or sample_state() or state
+                    state = sample_state() or state
 
                 except BaseException as ex:
                     error, raise_error = handle_error(ex)
@@ -735,9 +731,6 @@ async def task_run_sample(
                 if time_limit is not None:
                     timeout_cm = anyio.fail_after(time_limit / 2)
 
-                # turn off message and token limits
-                state.message_limit = None
-                state.token_limit = None
                 set_sample_state(state)
 
                 # scoring
@@ -929,7 +922,7 @@ async def log_sample(
         input=sample.input,
         choices=sample.choices,
         target=sample.target,
-        metadata=state.metadata if state.metadata else {},
+        metadata=sample.metadata or {},
         sandbox=sample.sandbox,
         files=list(sample.files.keys()) if sample.files else None,
         setup=sample.setup,