inspect-ai 0.3.69__py3-none-any.whl → 0.3.71__py3-none-any.whl

inspect_ai/_view/www/log-schema.json

@@ -8,6 +8,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -85,6 +89,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "message",
@@ -216,6 +221,9 @@
                   {
                     "$ref": "#/$defs/ContentText"
                   },
+                  {
+                    "$ref": "#/$defs/ContentReasoning"
+                  },
                   {
                     "$ref": "#/$defs/ContentImage"
                   },
@@ -262,26 +270,13 @@
           ],
           "default": null,
           "title": "Tool Calls"
-        },
-        "reasoning": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "null"
-            }
-          ],
-          "default": null,
-          "title": "Reasoning"
         }
       },
       "required": [
         "role",
         "content",
         "source",
-        "tool_calls",
-        "reasoning"
+        "tool_calls"
       ],
       "title": "ChatMessageAssistant",
       "type": "object",
@@ -307,6 +302,9 @@
                   {
                     "$ref": "#/$defs/ContentText"
                   },
+                  {
+                    "$ref": "#/$defs/ContentReasoning"
+                  },
                   {
                     "$ref": "#/$defs/ContentImage"
                   },
@@ -369,6 +367,9 @@
                   {
                     "$ref": "#/$defs/ContentText"
                   },
+                  {
+                    "$ref": "#/$defs/ContentReasoning"
+                  },
                   {
                     "$ref": "#/$defs/ContentImage"
                   },
@@ -469,6 +470,9 @@
                   {
                     "$ref": "#/$defs/ContentText"
                   },
+                  {
+                    "$ref": "#/$defs/ContentReasoning"
+                  },
                   {
                     "$ref": "#/$defs/ContentImage"
                   },
@@ -591,6 +595,47 @@
       "type": "object",
       "additionalProperties": false
     },
+    "ContentReasoning": {
+      "description": "Reasoning content.\n\nSee the specification for [thinking blocks](https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#understanding-thinking-blocks) for Claude models.",
+      "properties": {
+        "type": {
+          "const": "reasoning",
+          "default": "reasoning",
+          "title": "Type",
+          "type": "string"
+        },
+        "reasoning": {
+          "title": "Reasoning",
+          "type": "string"
+        },
+        "signature": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Signature"
+        },
+        "redacted": {
+          "default": false,
+          "title": "Redacted",
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "type",
+        "reasoning",
+        "signature",
+        "redacted"
+      ],
+      "title": "ContentReasoning",
+      "type": "object",
+      "additionalProperties": false
+    },
     "ContentText": {
       "description": "Text content.",
       "properties": {
@@ -653,6 +698,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -677,6 +726,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "error"
@@ -830,6 +880,18 @@
           "default": null,
           "title": "Time Limit"
         },
+        "working_limit": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Working Limit"
+        },
         "max_samples": {
           "anyOf": [
             {
@@ -951,6 +1013,7 @@
         "message_limit",
         "token_limit",
         "time_limit",
+        "working_limit",
         "max_samples",
         "max_tasks",
         "max_subprocesses",
@@ -1118,6 +1181,33 @@
       "type": "object",
       "additionalProperties": false
     },
+    "EvalMetricDefinition": {
+      "properties": {
+        "name": {
+          "title": "Name",
+          "type": "string"
+        },
+        "options": {
+          "anyOf": [
+            {
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Options"
+        }
+      },
+      "required": [
+        "name",
+        "options"
+      ],
+      "title": "EvalMetricDefinition",
+      "type": "object",
+      "additionalProperties": false
+    },
     "EvalPlan": {
       "description": "Plan (solvers) used in evaluation.",
       "properties": {
@@ -1170,6 +1260,7 @@
             "max_tool_output": null,
             "cache_prompt": null,
             "reasoning_effort": null,
+            "reasoning_tokens": null,
             "reasoning_history": null
           }
         }
@@ -1441,6 +1532,9 @@
               {
                 "$ref": "#/$defs/SampleLimitEvent"
               },
+              {
+                "$ref": "#/$defs/SandboxEvent"
+              },
               {
                 "$ref": "#/$defs/StateEvent"
               },
@@ -1489,6 +1583,42 @@
           "title": "Model Usage",
           "type": "object"
         },
+        "total_time": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Total Time"
+        },
+        "working_time": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Working Time"
+        },
+        "uuid": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Uuid"
+        },
         "error": {
           "anyOf": [
             {
@@ -1535,6 +1665,9 @@
         "store",
         "events",
         "model_usage",
+        "total_time",
+        "working_time",
+        "uuid",
         "error",
         "attachments",
         "limit"
@@ -1550,6 +1683,7 @@
           "enum": [
             "context",
             "time",
+            "working",
             "message",
             "token",
             "operator",
@@ -1790,6 +1924,84 @@
       "type": "object",
       "additionalProperties": false
     },
+    "EvalScorer": {
+      "properties": {
+        "name": {
+          "title": "Name",
+          "type": "string"
+        },
+        "options": {
+          "anyOf": [
+            {
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Options"
+        },
+        "metrics": {
+          "anyOf": [
+            {
+              "items": {
+                "anyOf": [
+                  {
+                    "$ref": "#/$defs/EvalMetricDefinition"
+                  },
+                  {
+                    "additionalProperties": {
+                      "items": {
+                        "$ref": "#/$defs/EvalMetricDefinition"
+                      },
+                      "type": "array"
+                    },
+                    "type": "object"
+                  }
+                ]
+              },
+              "type": "array"
+            },
+            {
+              "additionalProperties": {
+                "items": {
+                  "$ref": "#/$defs/EvalMetricDefinition"
+                },
+                "type": "array"
+              },
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Metrics"
+        },
+        "metadata": {
+          "anyOf": [
+            {
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Metadata"
+        }
+      },
+      "required": [
+        "name",
+        "options",
+        "metrics",
+        "metadata"
+      ],
+      "title": "EvalScorer",
+      "type": "object",
+      "additionalProperties": false
+    },
     "EvalSpec": {
       "description": "Eval target and configuration.",
       "properties": {
@@ -1939,6 +2151,45 @@
           ],
           "default": null,
           "title": "Metadata"
+        },
+        "scorers": {
+          "anyOf": [
+            {
+              "items": {
+                "$ref": "#/$defs/EvalScorer"
+              },
+              "type": "array"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Scorers"
+        },
+        "metrics": {
+          "anyOf": [
+            {
+              "items": {
+                "$ref": "#/$defs/EvalMetricDefinition"
+              },
+              "type": "array"
+            },
+            {
+              "additionalProperties": {
+                "items": {
+                  "$ref": "#/$defs/EvalMetricDefinition"
+                },
+                "type": "array"
+              },
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Metrics"
         }
       },
       "required": [
@@ -1961,7 +2212,9 @@
         "config",
         "revision",
         "packages",
-        "metadata"
+        "metadata",
+        "scorers",
+        "metrics"
       ],
       "title": "EvalSpec",
       "type": "object",
@@ -2277,10 +2530,28 @@
           "default": null,
           "title": "Reasoning Effort"
         },
+        "reasoning_tokens": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Reasoning Tokens"
+        },
         "reasoning_history": {
           "anyOf": [
             {
-              "type": "boolean"
+              "enum": [
+                "none",
+                "all",
+                "last",
+                "auto"
+              ],
+              "type": "string"
             },
             {
               "type": "null"
@@ -2315,6 +2586,7 @@
         "max_tool_output",
         "cache_prompt",
         "reasoning_effort",
+        "reasoning_tokens",
         "reasoning_history"
       ],
       "additionalProperties": false
@@ -2327,6 +2599,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -2363,6 +2639,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "source",
@@ -2380,6 +2657,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -2409,6 +2690,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "input",
@@ -2478,6 +2760,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -2502,6 +2788,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "message"
@@ -2662,11 +2949,24 @@
           },
           "title": "Response",
           "type": "object"
+        },
+        "time": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Time"
         }
       },
       "required": [
         "request",
-        "response"
+        "response",
+        "time"
       ],
       "title": "ModelCall",
       "type": "object",
@@ -2680,6 +2980,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -2789,10 +3093,36 @@
             }
           ],
           "default": null
+        },
+        "completed": {
+          "anyOf": [
+            {
+              "format": "date-time",
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Completed"
+        },
+        "working_time": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Working Time"
         }
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "model",
@@ -2803,7 +3133,9 @@
         "output",
         "error",
         "cache",
-        "call"
+        "call",
+        "completed",
+        "working_time"
       ],
       "title": "ModelEvent",
       "type": "object",
@@ -2925,6 +3257,18 @@
           ],
           "default": null,
           "title": "Input Tokens Cache Read"
+        },
+        "reasoning_tokens": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Reasoning Tokens"
         }
       },
       "title": "ModelUsage",
@@ -2934,7 +3278,8 @@
         "output_tokens",
         "total_tokens",
         "input_tokens_cache_write",
-        "input_tokens_cache_read"
+        "input_tokens_cache_read",
+        "reasoning_tokens"
       ],
       "additionalProperties": false
     },
@@ -3085,6 +3430,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -3112,6 +3461,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "sample",
@@ -3129,6 +3479,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -3151,6 +3505,7 @@
           "enum": [
             "message",
             "time",
+            "working",
             "token",
             "operator",
             "custom"
@@ -3177,6 +3532,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "type",
@@ -3213,6 +3569,152 @@
       ],
       "type": "array"
     },
+    "SandboxEvent": {
+      "description": "Sandbox execution or I/O",
+      "properties": {
+        "timestamp": {
+          "format": "date-time",
+          "title": "Timestamp",
+          "type": "string"
+        },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
+        "pending": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Pending"
+        },
+        "event": {
+          "const": "sandbox",
+          "default": "sandbox",
+          "title": "Event",
+          "type": "string"
+        },
+        "action": {
+          "enum": [
+            "exec",
+            "read_file",
+            "write_file"
+          ],
+          "title": "Action",
+          "type": "string"
+        },
+        "cmd": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Cmd"
+        },
+        "options": {
+          "anyOf": [
+            {
+              "additionalProperties": {
+                "$ref": "#/$defs/JsonValue"
+              },
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Options"
+        },
+        "file": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "File"
+        },
+        "input": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Input"
+        },
+        "result": {
+          "anyOf": [
+            {
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Result"
+        },
+        "output": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Output"
+        },
+        "completed": {
+          "anyOf": [
+            {
+              "format": "date-time",
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Completed"
+        }
+      },
+      "required": [
+        "timestamp",
+        "working_start",
+        "pending",
+        "event",
+        "action",
+        "cmd",
+        "options",
+        "file",
+        "input",
+        "result",
+        "output",
+        "completed"
+      ],
+      "title": "SandboxEvent",
+      "type": "object",
+      "additionalProperties": false
+    },
     "Score": {
       "description": "Score generated by a scorer.",
       "properties": {
@@ -3329,6 +3831,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -3376,6 +3882,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "score",
@@ -3394,6 +3901,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -3422,6 +3933,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "changes"
@@ -3438,6 +3950,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -3483,6 +3999,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "action",
@@ -3501,6 +4018,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -3529,6 +4050,7 @@
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "changes"
@@ -3545,6 +4067,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -3596,6 +4122,9 @@
               {
                 "$ref": "#/$defs/SampleLimitEvent"
               },
+              {
+                "$ref": "#/$defs/SandboxEvent"
+              },
               {
                 "$ref": "#/$defs/StateEvent"
               },
@@ -3636,17 +4165,45 @@
           },
           "title": "Events",
           "type": "array"
+        },
+        "completed": {
+          "anyOf": [
+            {
+              "format": "date-time",
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Completed"
+        },
+        "working_time": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Working Time"
         }
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "name",
         "type",
         "input",
         "result",
-        "events"
+        "events",
+        "completed",
+        "working_time"
       ],
       "title": "SubtaskEvent",
       "type": "object",
@@ -3816,6 +4373,10 @@
           "title": "Timestamp",
           "type": "string"
         },
+        "working_start": {
+          "title": "Working Start",
+          "type": "number"
+        },
         "pending": {
           "anyOf": [
             {
@@ -3883,6 +4444,9 @@
             {
               "$ref": "#/$defs/ContentText"
             },
+            {
+              "$ref": "#/$defs/ContentReasoning"
+            },
             {
               "$ref": "#/$defs/ContentImage"
             },
@@ -3898,6 +4462,9 @@
                   {
                     "$ref": "#/$defs/ContentText"
                   },
+                  {
+                    "$ref": "#/$defs/ContentReasoning"
+                  },
                   {
                     "$ref": "#/$defs/ContentImage"
                   },
@@ -3956,6 +4523,9 @@
               {
                 "$ref": "#/$defs/SampleLimitEvent"
               },
+              {
+                "$ref": "#/$defs/SandboxEvent"
+              },
               {
                 "$ref": "#/$defs/StateEvent"
               },
@@ -3996,10 +4566,36 @@
           },
           "title": "Events",
           "type": "array"
+        },
+        "completed": {
+          "anyOf": [
+            {
+              "format": "date-time",
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Completed"
+        },
+        "working_time": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Working Time"
         }
       },
       "required": [
         "timestamp",
+        "working_start",
         "pending",
         "event",
         "type",
@@ -4010,7 +4606,9 @@
         "result",
         "truncated",
         "error",
-        "events"
+        "events",
+        "completed",
+        "working_time"
       ],
       "title": "ToolEvent",
       "type": "object",