inspect-ai 0.3.85__py3-none-any.whl → 0.3.86__py3-none-any.whl

inspect_ai/_eval/eval.py

@@ -43,7 +43,7 @@ from inspect_ai.model import (
     GenerateConfigArgs,
     Model,
 )
-from inspect_ai.model._model import init_active_model, resolve_models
+from inspect_ai.model._model import get_model, init_active_model, resolve_models
 from inspect_ai.scorer._reducer import reducer_log_names
 from inspect_ai.solver._chain import chain
 from inspect_ai.solver._solver import Solver, SolverSpec
@@ -751,10 +751,15 @@ async def eval_retry_async(
             else None
         )
 
+        # resolve the model
+        model = get_model(
+            model=eval_log.eval.model,
+            config=eval_log.eval.model_generate_config,
+            base_url=eval_log.eval.model_base_url,
+            **eval_log.eval.model_args,
+        )
+
         # collect the rest of the params we need for the eval
-        model = eval_log.eval.model
-        model_base_url = eval_log.eval.model_base_url
-        model_args = eval_log.eval.model_args
         task_args = eval_log.eval.task_args
         tags = eval_log.eval.tags
         limit = eval_log.eval.config.limit
@@ -813,8 +818,6 @@ async def eval_retry_async(
                     id=task_id, task=task, task_args=task_args, model=None, log=eval_log
                 ),
                 model=model,
-                model_base_url=model_base_url,
-                model_args=model_args,
                 task_args=task_args,
                 sandbox=eval_log.eval.sandbox,
                 sandbox_cleanup=sandbox_cleanup,

inspect_ai/_eval/task/log.py

@@ -139,6 +139,7 @@ class TaskLogger:
             tags=tags,
             solver_args=solver.args if solver else None,
             model=str(ModelName(model)),
+            model_generate_config=model.config,
             model_base_url=model.api.base_url,
             dataset=EvalDataset(
                 name=dataset.name,

inspect_ai/log/_log.py

@@ -599,6 +599,9 @@ class EvalSpec(BaseModel):
     model: str
     """Model used for eval."""
 
+    model_generate_config: GenerateConfig = Field(default_factory=GenerateConfig)
+    """Generate config specified for model instance."""
+
     model_base_url: str | None = Field(default=None)
     """Optional override of model base url"""
 

inspect_ai/log/_recorders/buffer/database.py

@@ -199,28 +199,36 @@ class SampleBufferDatabase(SampleBuffer):
             )
 
     def remove_samples(self, samples: list[tuple[str | int, int]]) -> None:
+        # short circuit no samples
+        if len(samples) == 0:
+            return
+
         with self._get_connection(write=True) as conn:
             cursor = conn.cursor()
             try:
-                # Convert list of tuples into a string for SQL IN clause
-                # Format: (('id1', 1), ('id2', 2))
-                sample_conditions = ",".join(
-                    [f"('{sid}', {epoch})" for sid, epoch in samples]
+                # Build a query using individual column comparisons instead of row values
+                placeholders = " OR ".join(
+                    ["(sample_id=? AND sample_epoch=?)" for _ in samples]
                 )
 
-                # Delete associated events first due to foreign key constraint
+                # Flatten parameters for binding
+                parameters = [item for tup in samples for item in tup]
+
+                # Delete associated events first
                 events_query = f"""
                     DELETE FROM events
-                    WHERE (sample_id, sample_epoch) IN ({sample_conditions})
+                    WHERE {placeholders}
                 """
-                cursor.execute(events_query)
+                cursor.execute(events_query, parameters)
+
+                # Then delete the samples using the same approach
+                placeholders = " OR ".join(["(id=? AND epoch=?)" for _ in samples])
 
-                # Then delete the samples
                 samples_query = f"""
                     DELETE FROM samples
-                    WHERE (id, epoch) IN ({sample_conditions})
+                    WHERE {placeholders}
                 """
-                cursor.execute(samples_query)
+                cursor.execute(samples_query, parameters)
             finally:
                 cursor.close()
 
@@ -259,7 +267,7 @@ class SampleBufferDatabase(SampleBuffer):
 
                 # fetch data
                 return Samples(
-                    samples=list(self._get_samples(conn)),
+                    samples=list(self._get_samples(conn, True)),
                     metrics=task_data.metrics,
                     refresh=self.update_interval,
                     etag=str(task_data.version),

inspect_ai/model/_openai.py

@@ -3,7 +3,7 @@ import re
 from copy import copy
 from typing import Literal
 
-from openai import BadRequestError, OpenAIError
+from openai import APIStatusError, OpenAIError
 from openai.types.chat import (
     ChatCompletion,
     ChatCompletionAssistantMessageParam,
@@ -518,7 +518,7 @@ def chat_choices_from_openai(
 
 
 def openai_handle_bad_request(
-    model_name: str, e: BadRequestError
+    model_name: str, e: APIStatusError
 ) -> ModelOutput | Exception:
     # extract message
     if isinstance(e.body, dict) and "message" in e.body.keys():

inspect_ai/model/_providers/openai.py

@@ -13,6 +13,7 @@ from openai import (
     AsyncOpenAI,
     BadRequestError,
     RateLimitError,
+    UnprocessableEntityError,
 )
 from openai._types import NOT_GIVEN
 from openai.types.chat import ChatCompletion
@@ -295,13 +296,13 @@ class OpenAIAPI(ModelAPI):
                     else None
                 ),
             ), model_call()
-        except BadRequestError as e:
+        except (BadRequestError, UnprocessableEntityError) as e:
             return self.handle_bad_request(e), model_call()
 
     def on_response(self, response: dict[str, Any]) -> None:
         pass
 
-    def handle_bad_request(self, ex: BadRequestError) -> ModelOutput | Exception:
+    def handle_bad_request(self, ex: APIStatusError) -> ModelOutput | Exception:
         return openai_handle_bad_request(self.model_name, ex)
 
     def _chat_choices_from_response(

inspect_ai/model/_providers/together.py

@@ -3,7 +3,7 @@ from json import dumps
 from typing import Any
 
 import httpx
-from openai import BadRequestError
+from openai import APIStatusError
 from openai.types.chat import (
     ChatCompletion,
 )
@@ -105,7 +105,7 @@ class TogetherAIAPI(OpenAIAPI):
         return DEFAULT_MAX_TOKENS
 
     @override
-    def handle_bad_request(self, ex: BadRequestError) -> ModelOutput | Exception:
+    def handle_bad_request(self, ex: APIStatusError) -> ModelOutput | Exception:
         response = ex.response.json()
         if "error" in response and "message" in response.get("error"):
             content = response.get("error").get("message")

{inspect_ai-0.3.85.dist-info → inspect_ai-0.3.86.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: inspect_ai
-Version: 0.3.85
+Version: 0.3.86
 Summary: Framework for large language model evaluations
 Author: UK AI Security Institute
 License: MIT License

{inspect_ai-0.3.85.dist-info → inspect_ai-0.3.86.dist-info}/RECORD

@@ -46,7 +46,7 @@ inspect_ai/_display/textual/widgets/transcript.py,sha256=zaxlDixT6Fie0acAWBM9Hlt
 inspect_ai/_display/textual/widgets/vscode.py,sha256=YTXdIZ0fcf9XE2v3rWIfUTgnXFww8uKCo7skugQLIbs,1247
 inspect_ai/_eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 inspect_ai/_eval/context.py,sha256=gWTjEEMVTJMJpCCKLRs4joZDkG00rzE7-HXZFyzSC_I,1283
-inspect_ai/_eval/eval.py,sha256=3VHOUYhkTmni-AaT4dYE9kfoXME68eCqOv_T6xFySzo,40266
+inspect_ai/_eval/eval.py,sha256=tfIYOJSNGNfZHl18XYqXZHIMKlCmUZ8gbqe7I0OZJII,40307
 inspect_ai/_eval/evalset.py,sha256=FnZBVi5hOt6f84PNYlFhkjb7N1lNgiQydQlernJZeW4,24005
 inspect_ai/_eval/list.py,sha256=VbZ-2EI6MqrXvCN7VTz21TQSoU5K5_Q0hqhxmj5A_m0,3744
 inspect_ai/_eval/loader.py,sha256=yCDrW5MhP6GT329hZ_gUm_eAMsCA9G7jb8sm45Pj-pw,24970
@@ -59,7 +59,7 @@ inspect_ai/_eval/task/epochs.py,sha256=Ci7T6CQniSOTChv5Im2dCdSDrP-5hq19rV6iJ2uBc
 inspect_ai/_eval/task/error.py,sha256=Vhqinfdf0eIrjn7kUY7-id8Kbdggr-fEFpAJeJrkJ1M,1244
 inspect_ai/_eval/task/generate.py,sha256=C9-S9ak4VFQO7QgtUbGjt8F4sTyXS5nekR3Mg_MPwmM,2511
 inspect_ai/_eval/task/images.py,sha256=nTzHizlyuPYumPH7gAOBSrNkTwTbAmZ7tKdzN7d_R2k,4035
-inspect_ai/_eval/task/log.py,sha256=w1Uu3VplvL_UUqyCVDmUMOG5s8_E3si6OkglE7xPxM0,11769
+inspect_ai/_eval/task/log.py,sha256=PD2ZrqtHY0zRyx7pB8L5v-txyaBRePs76cFu5Fb-vjE,11817
 inspect_ai/_eval/task/resolved.py,sha256=OCQc_0HmW_Vw8o1KisX0DCn-eOPkTbR1v_y_jEaAlhU,966
 inspect_ai/_eval/task/results.py,sha256=x4weYRK2XGowfBG3f2msOeZQ_pxh230HTlw6kps33jw,17925
 inspect_ai/_eval/task/run.py,sha256=RS2Qv3AythSkQL4fsgBFaXfyx2WDIZuFj9v6ifoRiYs,38714
@@ -510,7 +510,7 @@ inspect_ai/log/_bundle.py,sha256=5Uy-s64_SFokZ7WRzti9mD7yoKrd2sOzdvqKyahoiC4,804
 inspect_ai/log/_condense.py,sha256=OedMphK5Q2YPuY1cnoAM7tGsyVIU6Kwrv3oIeb3dFmY,10881
 inspect_ai/log/_convert.py,sha256=qn6q10Um2XV7dnK4nQargANa0bz6RFJPmaEMINv38cs,3467
 inspect_ai/log/_file.py,sha256=QjeVUegoCWVUv6CMsj0das_UpZZZMfnbvCQAKlFYGXE,17105
-inspect_ai/log/_log.py,sha256=f4ChtLdNc_z0qVXsJCmZyW6BdbFKGTfHWY5gaymsUkc,24970
+inspect_ai/log/_log.py,sha256=KsssY2kGfuDHGIXOGJHN4bO1LXVs0f3XtqIUfA2R68A,25109
 inspect_ai/log/_message.py,sha256=QofM_JZF_x3k_5ta1uQzoN_VnMoUhXFnqWurIn9FXOY,1999
 inspect_ai/log/_retry.py,sha256=e7a2hjl3Ncl8b8sU7CsDpvK8DV0b1uSRLeokRX1mt34,2109
 inspect_ai/log/_samples.py,sha256=wPQlV1VR9djWaj37lLrjBprCabdAm4S2vFOsQTcd12U,4910
@@ -524,7 +524,7 @@ inspect_ai/log/_recorders/recorder.py,sha256=zDDpl2tktPjb6xk5kd4TyEMxkXZiLgXXpPi
 inspect_ai/log/_recorders/types.py,sha256=Aeo-U7FhmWQSvE_uz3fwUI7cqaSR-ZE_uRVu-1fBCgc,865
 inspect_ai/log/_recorders/buffer/__init__.py,sha256=6DsRdnNl-ic-xJmnBE5i45ZP3eB4yAta9wxi5WFcbqc,367
 inspect_ai/log/_recorders/buffer/buffer.py,sha256=rtLvaX7nSqNrWb-3CeSaOHwJgF1CzRgXFT_I1dDkM1k,945
-inspect_ai/log/_recorders/buffer/database.py,sha256=aqBJdM6meQTWsLs9uF1gFGg1dsE1MvVQdiXR1DHoRqw,22171
+inspect_ai/log/_recorders/buffer/database.py,sha256=3yV8OlDsQ4zFQHNqe7aBAHwkUISW3zmaLBlD1OFj36w,22396
 inspect_ai/log/_recorders/buffer/filestore.py,sha256=S6RP-5zkOPSmy1hV2LCCbfwdX-YFZGuIEjfJuOWMjDQ,8274
 inspect_ai/log/_recorders/buffer/types.py,sha256=pTnPCZHbk9qF6yF-eNXHTa23cLH_FvP8dmfPJCFO15Q,2046
 inspect_ai/model/__init__.py,sha256=6Aa_HEU-rgxWPDaIRlE6KBdXY406x2LtcLeVtAxk-AI,2453
@@ -537,7 +537,7 @@ inspect_ai/model/_generate_config.py,sha256=_-kzw7LOl45baVkTjlfL1K1VLKGgNOOczH2H
 inspect_ai/model/_model.py,sha256=h4ASS2VuTZ_97145rLW202u6e7-mw4ENnnlBl0Vsbio,52127
 inspect_ai/model/_model_call.py,sha256=VJ8wnl9Y81JaiClBYM8eyt1jVb3n-yc6Dd88ofRiJDc,2234
 inspect_ai/model/_model_output.py,sha256=R5EAUPLc5RWymVb3le4cbqbNCZ9voTzg0U1j_e4I-yM,7768
-inspect_ai/model/_openai.py,sha256=0OAmxQbIU6V7WJr9Q8J6oGwQuY9aZLPpHQ9r28GCmbg,19382
+inspect_ai/model/_openai.py,sha256=-N_LhZR8-nrnCL8h9lklo_RrGNDR1SzMJ0tPafVuPXo,19380
 inspect_ai/model/_openai_computer_use.py,sha256=vbKkYLhqNuX16zuWfg5MaGp9H8URrPcLhKQ1pDsZtPo,5943
 inspect_ai/model/_openai_responses.py,sha256=bQWuVvJIkS8CqtoX9z1aRb1aky4TNbMngG2paB3wsrA,20179
 inspect_ai/model/_reasoning.py,sha256=qmR8WT6t_cb7NIsJOQHPyFZh2eLV0HmYxKo2vtvteQ4,929
@@ -556,12 +556,12 @@ inspect_ai/model/_providers/mistral.py,sha256=FbMPN_pw8LZal2iFGf5FX70ypuH3k44FUn
 inspect_ai/model/_providers/mockllm.py,sha256=gL9f-f5TOdE4a0GVENr3cOIIp2kv8zVXWPZ608rouGk,2440
 inspect_ai/model/_providers/none.py,sha256=6qLbZpHSoEZaaxFO7luieFjqig2Ju8Fu00DlRngAry8,935
 inspect_ai/model/_providers/ollama.py,sha256=mBPSxaEkiH_RnlHKqOyFBlXObQhc2dfjL-rCKrea5u8,675
-inspect_ai/model/_providers/openai.py,sha256=NFdMpnI2vlmpI8h_vWnt8y4X_XaydaL9gH5Dmy6k5Tw,16478
+inspect_ai/model/_providers/openai.py,sha256=zJkhtiEQrmsuhfL7mpBPpOlYJ_WNraeyTkjYTelF0no,16535
 inspect_ai/model/_providers/openai_o1.py,sha256=k-Xm_Wzn1KHKL6Z1KTHg4CTTr8ybgiHvXkLiLdjP7Os,12926
 inspect_ai/model/_providers/openai_responses.py,sha256=YPXt8KQfIEiiTpvtoQECBoNQLDLbwBW_KhBfM8vEhJk,6324
 inspect_ai/model/_providers/openrouter.py,sha256=pDimDmm_4FzS4GZx0n9z8z717mQf3IQlgEy30huzpc4,4730
 inspect_ai/model/_providers/providers.py,sha256=Sd2D9OcWkukuBcl_-KDfdpxMaAShv1JZhL5KfAM87CE,5817
-inspect_ai/model/_providers/together.py,sha256=MoA3tyMKUnE0EekTqEIBBwvsaOp5c697kydLi1ZMYzE,9745
+inspect_ai/model/_providers/together.py,sha256=Wh3G0vhKHq5ofx1otwXjJFhM98Ll70IbqBhUNNV2-rk,9743
 inspect_ai/model/_providers/vertex.py,sha256=60W7kgoA83GtKdMeJgNU2IAw0N0wTscg4YCcMPu2bwo,17185
 inspect_ai/model/_providers/vllm.py,sha256=UYjCCXzw2hGJHVC3oPl-u2EI4iAm8ZncoIfYp1QJkbQ,14238
 inspect_ai/model/_providers/util/__init__.py,sha256=d4T_qvXihTRd1zmQkNE3xUBlHCX8tOIbRK19EwU0fTs,717
@@ -692,9 +692,9 @@ inspect_ai/util/_sandbox/docker/internal.py,sha256=c8X8TLrBPOvsfnq5TkMlb_bzTALyc
 inspect_ai/util/_sandbox/docker/prereqs.py,sha256=0j6_OauBBnVlpBleADcZavIAAQZy4WewVjbRn9c0stg,3355
 inspect_ai/util/_sandbox/docker/service.py,sha256=hhHIWH1VDFLwehdGd19aUBD_VKfDO3GCPxpw1HSwVQk,2437
 inspect_ai/util/_sandbox/docker/util.py,sha256=EeInihCNXgUWxaqZ4dNOJd719kXL2_jr63QCoXn68vA,3154
-inspect_ai-0.3.85.dist-info/licenses/LICENSE,sha256=xZPCr8gTiFIerrA_DRpLAbw-UUftnLFsHxKeW-NTtq8,1081
-inspect_ai-0.3.85.dist-info/METADATA,sha256=yNjYFfYdXBQnzqg-_uT3awl6p1mr9eOKlJrIOIPtCZs,4965
-inspect_ai-0.3.85.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-inspect_ai-0.3.85.dist-info/entry_points.txt,sha256=WGGLmzTzDWLzYfiyovSY6oEKuf-gqzSDNOb5V-hk3fM,54
-inspect_ai-0.3.85.dist-info/top_level.txt,sha256=Tp3za30CHXJEKLk8xLe9qGsW4pBzJpEIOMHOHNCXiVo,11
-inspect_ai-0.3.85.dist-info/RECORD,,
+inspect_ai-0.3.86.dist-info/licenses/LICENSE,sha256=xZPCr8gTiFIerrA_DRpLAbw-UUftnLFsHxKeW-NTtq8,1081
+inspect_ai-0.3.86.dist-info/METADATA,sha256=tId3lj5ywe2A79iWACtXXP_aBeWfnbQQfAujxJGxaoc,4965
+inspect_ai-0.3.86.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+inspect_ai-0.3.86.dist-info/entry_points.txt,sha256=WGGLmzTzDWLzYfiyovSY6oEKuf-gqzSDNOb5V-hk3fM,54
+inspect_ai-0.3.86.dist-info/top_level.txt,sha256=Tp3za30CHXJEKLk8xLe9qGsW4pBzJpEIOMHOHNCXiVo,11
+inspect_ai-0.3.86.dist-info/RECORD,,