inspect-ai 0.3.77__py3-none-any.whl → 0.3.79__py3-none-any.whl

inspect_ai/_eval/run.py

@@ -115,16 +115,6 @@ async def eval_run(
         eval_solver = None
         eval_solver_spec = None
 
-    # resolve the task scorers
-    eval_scorer_specs = (
-        [as_scorer_spec(scorer) for scorer in task.scorer]
-        if task.scorer is not None
-        else None
-    )
-
-    # resolve task metrics
-    eval_metrics = to_metric_specs(task.metrics) if task.metrics is not None else None
-
     try:
         # create run tasks
         task_run_options: list[TaskRunOptions] = []
@@ -137,6 +127,18 @@ async def eval_run(
                 task = resolved_task.task
                 task_eval_config = eval_config.model_copy()
 
+                # resolve the task scorers
+                eval_scorer_specs = (
+                    [as_scorer_spec(scorer) for scorer in task.scorer]
+                    if task.scorer is not None
+                    else None
+                )
+
+                # resolve task metrics
+                eval_metrics = (
+                    to_metric_specs(task.metrics) if task.metrics is not None else None
+                )
+
                 # epochs
                 if task_eval_config.epochs is None:
                     task_eval_config.epochs = task.epochs

inspect_ai/model/_openai_responses.py

@@ -73,11 +73,17 @@ async def openai_responses_input(
     elif message.role == "assistant":
         reasoning_content = openai_responses_reasponing_content_params(message.content)
         if message.content:
+            formatted_id = str(message.id).replace("resp_", "msg_", 1)
+            if not formatted_id.startswith("msg_"):
+                # These messages MUST start with `msg_`.
+                # As `store=False` for this provider, OpenAI doesn't validate the IDs.
+                # This will keep them consistent across calls though.
+                formatted_id = f"msg_{formatted_id}"
             text_content = [
                 ResponseOutputMessageParam(
                     type="message",
                     role="assistant",
-                    id=str(message.id).replace("resp_", "msg_", 1),
+                    id=formatted_id,
                     content=openai_responses_text_content_params(message.content),
                     status="completed",
                 )

inspect_ai/model/_providers/google.py

@@ -177,15 +177,10 @@ class GoogleGenAIAPI(ModelAPI):
                 self.api_key = os.environ.get(GOOGLE_API_KEY, None)
 
             # custom base_url
-            base_url = model_base_url(base_url, "GOOGLE_BASE_URL")
+            self.base_url = model_base_url(self.base_url, "GOOGLE_BASE_URL")
 
-        # create client
-        self.client = Client(
-            vertexai=self.is_vertex(),
-            api_key=self.api_key,
-            http_options={"base_url": base_url},
-            **model_args,
-        )
+        # save model args
+        self.model_args = model_args
 
     @override
     async def close(self) -> None:
@@ -202,11 +197,19 @@ class GoogleGenAIAPI(ModelAPI):
         tool_choice: ToolChoice,
         config: GenerateConfig,
     ) -> ModelOutput | tuple[ModelOutput | Exception, ModelCall]:
+        # create client
+        client = Client(
+            vertexai=self.is_vertex(),
+            api_key=self.api_key,
+            http_options={"base_url": self.base_url},
+            **self.model_args,
+        )
+
         # generate request_id
         request_id = urllib3_hooks().start_request()
 
         # Create google-genai types.
-        gemini_contents = await as_chat_messages(self.client, input)
+        gemini_contents = await as_chat_messages(client, input)
         gemini_tools = chat_tools(tools) if len(tools) > 0 else None
         gemini_tool_config = chat_tool_config(tool_choice) if len(tools) > 0 else None
         parameters = GenerateContentConfig(
@@ -222,9 +225,7 @@ class GoogleGenAIAPI(ModelAPI):
             safety_settings=safety_settings_to_list(self.safety_settings),
             tools=gemini_tools,
             tool_config=gemini_tool_config,
-            system_instruction=await extract_system_message_as_parts(
-                self.client, input
-            ),
+            system_instruction=await extract_system_message_as_parts(client, input),
         )
         if config.response_schema is not None:
             parameters.response_mime_type = "application/json"
@@ -246,7 +247,7 @@ class GoogleGenAIAPI(ModelAPI):
             )
 
         try:
-            response = await self.client.aio.models.generate_content(
+            response = await client.aio.models.generate_content(
                 model=self.model_name,
                 contents=gemini_contents,
                 config=parameters,

{inspect_ai-0.3.77.dist-info → inspect_ai-0.3.79.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: inspect_ai
-Version: 0.3.77
+Version: 0.3.79
 Summary: Framework for large language model evaluations
 Author: UK AI Security Institute
 License: MIT License

{inspect_ai-0.3.77.dist-info → inspect_ai-0.3.79.dist-info}/RECORD

@@ -50,7 +50,7 @@ inspect_ai/_eval/evalset.py,sha256=HGrz0LkTMsBbYDPZEMVnZCmFi_pYegZtSoqRVYbRDiE,2
 inspect_ai/_eval/list.py,sha256=VbZ-2EI6MqrXvCN7VTz21TQSoU5K5_Q0hqhxmj5A_m0,3744
 inspect_ai/_eval/loader.py,sha256=yOj8HqYBFQntx0_GY4Wxqm6jivlT4N4WiQ1T8J9uRVA,23606
 inspect_ai/_eval/registry.py,sha256=9Q-Re9uZagQ2nw-W7hA6zhrmCQFmo3KcxncTcG24EG8,5315
-inspect_ai/_eval/run.py,sha256=9TtOp1vc18KclxstjvWhEL5VtcLBL3SRbRwsoRXaah8,19179
+inspect_ai/_eval/run.py,sha256=7Cepcyd5KaMcoUQybRifTtFEOLBBvmszhJdH_-yzb3E,19315
 inspect_ai/_eval/score.py,sha256=qf9T8XwUmfE6A8QK1-D-jUbNlLETVM-HXhucPWZ0Ro0,9591
 inspect_ai/_eval/task/__init__.py,sha256=6FvojMW3yo36L7xDacppCHDxt6A8_tzj_ftg5bQ6eNk,199
 inspect_ai/_eval/task/constants.py,sha256=quAKMw-4-3xKd1T_KwXCZvHYoKRXt1ZGuaHbBcWJwnA,72
@@ -460,7 +460,7 @@ inspect_ai/model/_model.py,sha256=LaRXqKvM2fl9HGCQAdrOOkNryfYU970LZGsyOxk1o00,48
 inspect_ai/model/_model_call.py,sha256=VJ8wnl9Y81JaiClBYM8eyt1jVb3n-yc6Dd88ofRiJDc,2234
 inspect_ai/model/_model_output.py,sha256=1picjX2Y0NSmEZ-vdQi42QAx3QvMcWVRn8pXP2wtmN8,7733
 inspect_ai/model/_openai.py,sha256=3_lQ3u-WJ8BOY5bMLM49LvAePzCZAn57Vi2SNLrWPm4,19093
-inspect_ai/model/_openai_responses.py,sha256=7xViQD1JZuoAkT_0znFT1-r-q57m2OvL-zaEoWP1FYM,9690
+inspect_ai/model/_openai_responses.py,sha256=qav1Fj2R-DOCkUfkrtGxV_PeT578mBZNwKt-yL4bmT0,10049
 inspect_ai/model/_reasoning.py,sha256=qmR8WT6t_cb7NIsJOQHPyFZh2eLV0HmYxKo2vtvteQ4,929
 inspect_ai/model/_registry.py,sha256=Cr2y32EqLnOqLbSWoXHVK4ivTTzCUhJuACxoTyPt8kY,2032
 inspect_ai/model/_render.py,sha256=rWypNUjgrH4NGp0r-ESAze9gZz7lYNjheEP438vRYZE,922
@@ -469,7 +469,7 @@ inspect_ai/model/_providers/azureai.py,sha256=tDWuePLhnZBcpHLVzX7J3Wx8VRPhW8tmtj
 inspect_ai/model/_providers/bedrock.py,sha256=mLeMW2JkG4lF0VQGEdku73ZL00EBy-hEvEcyCdjDUDo,24153
 inspect_ai/model/_providers/cloudflare.py,sha256=0e0HPas21cVC8N9mNJlZnSZyIt6FUB9lTIAqPXJDrtE,4586
 inspect_ai/model/_providers/goodfire.py,sha256=EzebC1woEjIXfHLP_ixpMR6G1hC-LxbSUxiilq1c-Is,8868
-inspect_ai/model/_providers/google.py,sha256=YkgeFxVh9hMWoqiHe39wgt02eNTogMxmlYX6goYWyo0,28697
+inspect_ai/model/_providers/google.py,sha256=zWvANIFJfkrsKfXZXL37gEr318uIRWEP7t7nDC20efA,28731
 inspect_ai/model/_providers/grok.py,sha256=dS88ueXiD-kHAFr0jCoTpTGLGa2VsUlB_TFP8L_2lBM,995
 inspect_ai/model/_providers/groq.py,sha256=Fr4fy8NmqllmUW7jhnQ3W94zGlxyr276qaGFS_iDI3Q,11189
 inspect_ai/model/_providers/hf.py,sha256=EZRiiRSzIoRCdFYKj3Otn5ebsROdjzx5YSQ6CzqOJxk,17969
@@ -627,9 +627,9 @@ inspect_ai/util/_sandbox/docker/internal.py,sha256=c8X8TLrBPOvsfnq5TkMlb_bzTALyc
 inspect_ai/util/_sandbox/docker/prereqs.py,sha256=0j6_OauBBnVlpBleADcZavIAAQZy4WewVjbRn9c0stg,3355
 inspect_ai/util/_sandbox/docker/service.py,sha256=hhHIWH1VDFLwehdGd19aUBD_VKfDO3GCPxpw1HSwVQk,2437
 inspect_ai/util/_sandbox/docker/util.py,sha256=EeInihCNXgUWxaqZ4dNOJd719kXL2_jr63QCoXn68vA,3154
-inspect_ai-0.3.77.dist-info/licenses/LICENSE,sha256=xZPCr8gTiFIerrA_DRpLAbw-UUftnLFsHxKeW-NTtq8,1081
-inspect_ai-0.3.77.dist-info/METADATA,sha256=7neEk8lnUwF8p-Snz7AFFvKCnxMsClI7B8cTJ2Tpo44,4997
-inspect_ai-0.3.77.dist-info/WHEEL,sha256=DK49LOLCYiurdXXOXwGJm6U4DkHkg4lcxjhqwRa0CP4,91
-inspect_ai-0.3.77.dist-info/entry_points.txt,sha256=WGGLmzTzDWLzYfiyovSY6oEKuf-gqzSDNOb5V-hk3fM,54
-inspect_ai-0.3.77.dist-info/top_level.txt,sha256=Tp3za30CHXJEKLk8xLe9qGsW4pBzJpEIOMHOHNCXiVo,11
-inspect_ai-0.3.77.dist-info/RECORD,,
+inspect_ai-0.3.79.dist-info/licenses/LICENSE,sha256=xZPCr8gTiFIerrA_DRpLAbw-UUftnLFsHxKeW-NTtq8,1081
+inspect_ai-0.3.79.dist-info/METADATA,sha256=dT3qQqF-VjPdilNfzt9Lh8PhNbq66fckvnlGimNxAAY,4997
+inspect_ai-0.3.79.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+inspect_ai-0.3.79.dist-info/entry_points.txt,sha256=WGGLmzTzDWLzYfiyovSY6oEKuf-gqzSDNOb5V-hk3fM,54
+inspect_ai-0.3.79.dist-info/top_level.txt,sha256=Tp3za30CHXJEKLk8xLe9qGsW4pBzJpEIOMHOHNCXiVo,11
+inspect_ai-0.3.79.dist-info/RECORD,,

{inspect_ai-0.3.77.dist-info → inspect_ai-0.3.79.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.0.2)
+Generator: setuptools (78.1.0)
 Root-Is-Purelib: true
 Tag: py3-none-any