inspect-ai 0.3.70__py3-none-any.whl → 0.3.72__py3-none-any.whl

inspect_ai/_view/www/src/types/log.d.ts

@@ -103,7 +103,8 @@ export type InternalTools = boolean | null;
 export type MaxToolOutput = number | null;
 export type CachePrompt = "auto" | boolean | null;
 export type ReasoningEffort = ("low" | "medium" | "high") | null;
-export type ReasoningHistory = boolean | null;
+export type ReasoningTokens = number | null;
+export type ReasoningHistory = ("none" | "all" | "last" | "auto") | null;
 export type TotalSamples = number;
 export type CompletedSamples = number;
 export type Name5 = string;
@@ -122,6 +123,7 @@ export type OutputTokens = number;
 export type TotalTokens = number;
 export type InputTokensCacheWrite = number | null;
 export type InputTokensCacheRead = number | null;
+export type ReasoningTokens1 = number | null;
 export type Message = string;
 export type Traceback = string;
 export type TracebackAnsi = string;
@@ -139,47 +141,74 @@ export type Input =
 export type Role = "system";
 export type Content =
   | string
-  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
+  | (
+      | ContentText
+      | ContentReasoning
+      | ContentImage
+      | ContentAudio
+      | ContentVideo
+    )[];
 export type Type1 = "text";
 export type Text = string;
-export type Type2 = "image";
+export type Type2 = "reasoning";
+export type Reasoning = string;
+export type Signature = string | null;
+export type Redacted = boolean;
+export type Type3 = "image";
 export type Image = string;
 export type Detail = "auto" | "low" | "high";
-export type Type3 = "audio";
+export type Type4 = "audio";
 export type Audio = string;
 export type Format = "wav" | "mp3";
-export type Type4 = "video";
+export type Type5 = "video";
 export type Video = string;
 export type Format1 = "mp4" | "mpeg" | "mov";
 export type Source = ("input" | "generate") | null;
 export type Role1 = "user";
 export type Content1 =
   | string
-  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
+  | (
+      | ContentText
+      | ContentReasoning
+      | ContentImage
+      | ContentAudio
+      | ContentVideo
+    )[];
 export type Source1 = ("input" | "generate") | null;
 export type ToolCallId = string[] | null;
 export type Role2 = "assistant";
 export type Content2 =
   | string
-  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
+  | (
+      | ContentText
+      | ContentReasoning
+      | ContentImage
+      | ContentAudio
+      | ContentVideo
+    )[];
 export type Source2 = ("input" | "generate") | null;
 export type ToolCalls = ToolCall[] | null;
 export type Id1 = string;
 export type Function = string;
-export type Type5 = "function";
+export type Type6 = "function";
 export type ParseError = string | null;
 export type Title = string | null;
 export type Format2 = "text" | "markdown";
 export type Content3 = string;
-export type Reasoning = string | null;
 export type Role3 = "tool";
 export type Content4 =
   | string
-  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
+  | (
+      | ContentText
+      | ContentReasoning
+      | ContentImage
+      | ContentAudio
+      | ContentVideo
+    )[];
 export type Source3 = ("input" | "generate") | null;
 export type ToolCallId1 = string | null;
 export type Function1 = string | null;
-export type Type6 =
+export type Type7 =
   | "parsing"
   | "timeout"
   | "unicode_decode"
@@ -235,6 +264,7 @@ export type Answer = string | null;
 export type Explanation = string | null;
 export type Metadata6 = {} | null;
 export type Timestamp = string;
+export type WorkingStart = number;
 export type Pending = boolean | null;
 export type Event = "sample_init";
 export type Input1 =
@@ -255,9 +285,10 @@ export type Files1 = {
 export type Setup1 = string | null;
 export type JsonValue = unknown;
 export type Timestamp1 = string;
+export type WorkingStart1 = number;
 export type Pending1 = boolean | null;
 export type Event1 = "sample_limit";
-export type Type7 =
+export type Type8 =
   | "message"
   | "time"
   | "working"
@@ -267,6 +298,7 @@ export type Type7 =
 export type Message2 = string;
 export type Limit1 = number | null;
 export type Timestamp2 = string;
+export type WorkingStart2 = number;
 export type Pending2 = boolean | null;
 export type Event2 = "sandbox";
 export type Action = "exec" | "read_file" | "write_file";
@@ -278,7 +310,9 @@ export type File = string | null;
 export type Input2 = string | null;
 export type Result = number | null;
 export type Output = string | null;
+export type Completed = string | null;
 export type Timestamp3 = string;
+export type WorkingStart3 = number;
 export type Pending3 = boolean | null;
 export type Event3 = "state";
 export type Op = "remove" | "add" | "replace" | "move" | "test" | "copy";
@@ -286,10 +320,12 @@ export type Path = string;
 export type From = string | null;
 export type Changes = JsonChange[];
 export type Timestamp4 = string;
+export type WorkingStart4 = number;
 export type Pending4 = boolean | null;
 export type Event4 = "store";
 export type Changes1 = JsonChange[];
 export type Timestamp5 = string;
+export type WorkingStart5 = number;
 export type Pending5 = boolean | null;
 export type Event5 = "model";
 export type Model2 = string;
@@ -301,8 +337,8 @@ export type Input3 = (
 )[];
 export type Name7 = string;
 export type Description = string;
-export type Type8 = "object";
-export type Type9 =
+export type Type9 = "object";
+export type Type10 =
   | ("string" | "integer" | "number" | "boolean" | "array" | "object" | "null")
   | null;
 export type Description1 = string | null;
@@ -321,10 +357,13 @@ export type Name8 = string;
 export type Error1 = string | null;
 export type Cache = ("read" | "write") | null;
 export type Time1 = number | null;
+export type Completed1 = string | null;
+export type WorkingTime = number | null;
 export type Timestamp6 = string;
+export type WorkingStart6 = number;
 export type Pending6 = boolean | null;
 export type Event6 = "tool";
-export type Type10 = "function";
+export type Type11 = "function";
 export type Id3 = string;
 export type Function2 = string;
 export type Result1 =
@@ -332,12 +371,20 @@ export type Result1 =
   | number
   | boolean
   | ContentText
+  | ContentReasoning
   | ContentImage
   | ContentAudio
   | ContentVideo
-  | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
+  | (
+      | ContentText
+      | ContentReasoning
+      | ContentImage
+      | ContentAudio
+      | ContentVideo
+    )[];
 export type Truncated = [unknown, unknown] | null;
 export type Timestamp7 = string;
+export type WorkingStart7 = number;
 export type Pending7 = boolean | null;
 export type Event7 = "approval";
 export type Message3 = string;
@@ -350,19 +397,23 @@ export type Decision =
   | "terminate";
 export type Explanation1 = string | null;
 export type Timestamp8 = string;
+export type WorkingStart8 = number;
 export type Pending8 = boolean | null;
 export type Event8 = "input";
 export type Input4 = string;
 export type InputAnsi = string;
 export type Timestamp9 = string;
+export type WorkingStart9 = number;
 export type Pending9 = boolean | null;
 export type Event9 = "score";
 export type Target2 = string | string[] | null;
 export type Intermediate = boolean;
 export type Timestamp10 = string;
+export type WorkingStart10 = number;
 export type Pending10 = boolean | null;
 export type Event10 = "error";
 export type Timestamp11 = string;
+export type WorkingStart11 = number;
 export type Pending11 = boolean | null;
 export type Event11 = "logger";
 export type Name9 = string | null;
@@ -381,20 +432,23 @@ export type Filename = string;
 export type Module = string;
 export type Lineno = number;
 export type Timestamp12 = string;
+export type WorkingStart12 = number;
 export type Pending12 = boolean | null;
 export type Event12 = "info";
 export type Source4 = string | null;
 export type Timestamp13 = string;
+export type WorkingStart13 = number;
 export type Pending13 = boolean | null;
 export type Event13 = "step";
 export type Action1 = "begin" | "end";
-export type Type11 = string | null;
+export type Type12 = string | null;
 export type Name10 = string;
 export type Timestamp14 = string;
+export type WorkingStart14 = number;
 export type Pending14 = boolean | null;
 export type Event14 = "subtask";
 export type Name11 = string;
-export type Type12 = string | null;
+export type Type13 = string | null;
 export type Events2 = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -412,6 +466,8 @@ export type Events2 = (
   | StepEvent
   | SubtaskEvent
 )[];
+export type Completed2 = string | null;
+export type WorkingTime1 = number | null;
 export type Events1 = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -429,6 +485,8 @@ export type Events1 = (
   | StepEvent
   | SubtaskEvent
 )[];
+export type Completed3 = string | null;
+export type WorkingTime2 = number | null;
 export type Events = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -447,8 +505,9 @@ export type Events = (
   | SubtaskEvent
 )[];
 export type TotalTime = number | null;
-export type WorkingTime = number | null;
-export type Type13 =
+export type WorkingTime3 = number | null;
+export type Uuid = string | null;
+export type Type14 =
   | "context"
   | "time"
   | "working"
@@ -643,6 +702,7 @@ export interface GenerateConfig {
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
+  reasoning_tokens: ReasoningTokens;
   reasoning_history: ReasoningHistory;
 }
 /**
@@ -699,6 +759,7 @@ export interface ModelUsage1 {
   total_tokens: TotalTokens;
   input_tokens_cache_write: InputTokensCacheWrite;
   input_tokens_cache_read: InputTokensCacheRead;
+  reasoning_tokens: ReasoningTokens1;
 }
 /**
  * Eval error details.
@@ -728,7 +789,8 @@ export interface EvalSample {
   events: Events;
   model_usage: ModelUsage2;
   total_time: TotalTime;
-  working_time: WorkingTime;
+  working_time: WorkingTime3;
+  uuid: Uuid;
   error: EvalError | null;
   attachments: Attachments;
   limit: EvalSampleLimit | null;
@@ -748,11 +810,22 @@ export interface ContentText {
   type: Type1;
   text: Text;
 }
+/**
+ * Reasoning content.
+ *
+ * See the specification for [thinking blocks](https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking#understanding-thinking-blocks) for Claude models.
+ */
+export interface ContentReasoning {
+  type: Type2;
+  reasoning: Reasoning;
+  signature: Signature;
+  redacted: Redacted;
+}
 /**
  * Image content.
  */
 export interface ContentImage {
-  type: Type2;
+  type: Type3;
   image: Image;
   detail: Detail;
 }
@@ -760,7 +833,7 @@ export interface ContentImage {
  * Audio content.
  */
 export interface ContentAudio {
-  type: Type3;
+  type: Type4;
   audio: Audio;
   format: Format;
 }
@@ -768,7 +841,7 @@ export interface ContentAudio {
  * Video content.
  */
 export interface ContentVideo {
-  type: Type4;
+  type: Type5;
   video: Video;
   format: Format1;
 }
@@ -789,13 +862,12 @@ export interface ChatMessageAssistant {
   content: Content2;
   source: Source2;
   tool_calls: ToolCalls;
-  reasoning: Reasoning;
 }
 export interface ToolCall {
   id: Id1;
   function: Function;
   arguments: Arguments;
-  type: Type5;
+  type: Type6;
   parse_error: ParseError;
   view: ToolCallContent | null;
 }
@@ -820,7 +892,7 @@ export interface ChatMessageTool {
   error: ToolCallError | null;
 }
 export interface ToolCallError {
-  type: Type6;
+  type: Type7;
   message: Message1;
 }
 /**
@@ -881,6 +953,7 @@ export interface Store {}
  */
 export interface SampleInitEvent {
   timestamp: Timestamp;
+  working_start: WorkingStart;
   pending: Pending;
   event: Event;
   sample: Sample;
@@ -904,9 +977,10 @@ export interface Sample {
  */
 export interface SampleLimitEvent {
   timestamp: Timestamp1;
+  working_start: WorkingStart1;
   pending: Pending1;
   event: Event1;
-  type: Type7;
+  type: Type8;
   message: Message2;
   limit: Limit1;
 }
@@ -915,6 +989,7 @@ export interface SampleLimitEvent {
  */
 export interface SandboxEvent {
   timestamp: Timestamp2;
+  working_start: WorkingStart2;
   pending: Pending2;
   event: Event2;
   action: Action;
@@ -924,12 +999,14 @@ export interface SandboxEvent {
   input: Input2;
   result: Result;
   output: Output;
+  completed: Completed;
 }
 /**
  * Change to the current `TaskState`
  */
 export interface StateEvent {
   timestamp: Timestamp3;
+  working_start: WorkingStart3;
   pending: Pending3;
   event: Event3;
   changes: Changes;
@@ -953,6 +1030,7 @@ export interface JsonChange {
  */
 export interface StoreEvent {
   timestamp: Timestamp4;
+  working_start: WorkingStart4;
   pending: Pending4;
   event: Event4;
   changes: Changes1;
@@ -962,6 +1040,7 @@ export interface StoreEvent {
  */
 export interface ModelEvent {
   timestamp: Timestamp5;
+  working_start: WorkingStart5;
   pending: Pending5;
   event: Event5;
   model: Model2;
@@ -973,6 +1052,8 @@ export interface ModelEvent {
   error: Error1;
   cache: Cache;
   call: ModelCall | null;
+  completed: Completed1;
+  working_time: WorkingTime;
 }
 /**
  * Specification of a tool (JSON Schema compatible)
@@ -1009,7 +1090,7 @@ export interface ToolInfo {
  * Description of tool parameters object in JSON Schema format.
  */
 export interface ToolParams {
-  type: Type8;
+  type: Type9;
   properties: Properties;
   required: Required1;
   additionalProperties: Additionalproperties1;
@@ -1021,7 +1102,7 @@ export interface Properties {
  * Description of tool parameter in JSON Schema format.
  */
 export interface ToolParam {
-  type: Type9;
+  type: Type10;
   description: Description1;
   default: Default;
   enum: Enum;
@@ -1063,6 +1144,7 @@ export interface GenerateConfig1 {
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
+  reasoning_tokens: ReasoningTokens;
   reasoning_history: ReasoningHistory;
 }
 /**
@@ -1084,9 +1166,10 @@ export interface Response {
  */
 export interface ToolEvent {
   timestamp: Timestamp6;
+  working_start: WorkingStart6;
   pending: Pending6;
   event: Event6;
-  type: Type10;
+  type: Type11;
   id: Id3;
   function: Function2;
   arguments: Arguments1;
@@ -1095,6 +1178,8 @@ export interface ToolEvent {
   truncated: Truncated;
   error: ToolCallError | null;
   events: Events1;
+  completed: Completed3;
+  working_time: WorkingTime2;
 }
 export interface Arguments1 {
   [k: string]: JsonValue;
@@ -1104,6 +1189,7 @@ export interface Arguments1 {
  */
 export interface ApprovalEvent {
   timestamp: Timestamp7;
+  working_start: WorkingStart7;
   pending: Pending7;
   event: Event7;
   message: Message3;
@@ -1129,6 +1215,7 @@ export interface ToolCallView {
  */
 export interface InputEvent {
   timestamp: Timestamp8;
+  working_start: WorkingStart8;
   pending: Pending8;
   event: Event8;
   input: Input4;
@@ -1142,6 +1229,7 @@ export interface InputEvent {
  */
 export interface ScoreEvent {
   timestamp: Timestamp9;
+  working_start: WorkingStart9;
   pending: Pending9;
   event: Event9;
   score: Score;
@@ -1153,6 +1241,7 @@ export interface ScoreEvent {
  */
 export interface ErrorEvent {
   timestamp: Timestamp10;
+  working_start: WorkingStart10;
   pending: Pending10;
   event: Event10;
   error: EvalError;
@@ -1162,6 +1251,7 @@ export interface ErrorEvent {
  */
 export interface LoggerEvent {
   timestamp: Timestamp11;
+  working_start: WorkingStart11;
   pending: Pending11;
   event: Event11;
   message: LoggingMessage;
@@ -1183,6 +1273,7 @@ export interface LoggingMessage {
  */
 export interface InfoEvent {
   timestamp: Timestamp12;
+  working_start: WorkingStart12;
   pending: Pending12;
   event: Event12;
   source: Source4;
@@ -1193,10 +1284,11 @@ export interface InfoEvent {
  */
 export interface StepEvent {
   timestamp: Timestamp13;
+  working_start: WorkingStart13;
   pending: Pending13;
   event: Event13;
   action: Action1;
-  type: Type11;
+  type: Type12;
   name: Name10;
 }
 /**
@@ -1204,13 +1296,16 @@ export interface StepEvent {
  */
 export interface SubtaskEvent {
   timestamp: Timestamp14;
+  working_start: WorkingStart14;
   pending: Pending14;
   event: Event14;
   name: Name11;
-  type: Type12;
+  type: Type13;
   input: Input5;
   result: Result2;
   events: Events2;
+  completed: Completed2;
+  working_time: WorkingTime1;
 }
 export interface Input5 {}
 export interface Result2 {
@@ -1226,7 +1321,7 @@ export interface Attachments {
  * Limit encontered by sample.
  */
 export interface EvalSampleLimit {
-  type: Type13;
+  type: Type14;
   limit: Limit2;
 }
 /**

inspect_ai/_view/www/src/usage/ModelTokenTable.tsx

@@ -1,11 +1,13 @@
+import { FC } from "react";
+import { ModelUsage, ModelUsage2 } from "../types/log";
 import { TokenHeader, TokenRow, TokenTable } from "./TokenTable";
 
-interface ModelTokenTable {
-  model_usage: any;
+interface ModelTokenTableProps {
+  model_usage: ModelUsage | ModelUsage2;
   className?: string | string[];
 }
 
-export const ModelTokenTable: React.FC<ModelTokenTable> = ({
+export const ModelTokenTable: FC<ModelTokenTableProps> = ({
   model_usage,
   className,
 }) => {
@@ -14,13 +16,7 @@ export const ModelTokenTable: React.FC<ModelTokenTable> = ({
       <TokenHeader />
       <tbody>
         {Object.keys(model_usage).map((key) => {
-          return (
-            <TokenRow
-              key={key}
-              model={`${key}-token-row`}
-              usage={model_usage[key]}
-            />
-          );
+          return <TokenRow key={key} model={key} usage={model_usage[key]} />;
         })}
       </tbody>
     </TokenTable>

inspect_ai/_view/www/src/usage/ModelUsagePanel.module.css

@@ -22,3 +22,7 @@
   height: 1px;
   background-color: var(--bs-light-border-subtle);
 }
+
+.padded {
+  margin-bottom: 1em;
+}

inspect_ai/_view/www/src/usage/ModelUsagePanel.tsx

@@ -1,5 +1,5 @@
 import clsx from "clsx";
-import { Fragment } from "react";
+import { FC, Fragment } from "react";
 import { ModelUsage1 } from "../types/log";
 import { formatNumber } from "../utils/format";
 import styles from "./ModelUsagePanel.module.css";
@@ -13,23 +13,40 @@ interface ModelUsageRow {
   value?: number;
   secondary?: boolean;
   bordered?: boolean;
+  padded?: boolean;
 }
 
 /**
  * Renders the ModelUsagePanel component.
  */
-export const ModelUsagePanel: React.FC<ModelUsageProps> = ({ usage }) => {
+export const ModelUsagePanel: FC<ModelUsageProps> = ({ usage }) => {
   if (!usage) {
     return null;
   }
 
-  const rows: ModelUsageRow[] = [
-    {
-      label: "input",
-      value: usage.input_tokens,
+  const rows: ModelUsageRow[] = [];
+
+  if (usage.reasoning_tokens) {
+    rows.push({
+      label: "Reasoning",
+      value: usage.reasoning_tokens,
+      secondary: false,
+      bordered: true,
+    });
+
+    rows.push({
+      label: "---",
+      value: undefined,
       secondary: false,
-    },
-  ];
+      padded: true,
+    });
+  }
+
+  rows.push({
+    label: "input",
+    value: usage.input_tokens,
+    secondary: false,
+  });
 
   if (usage.input_tokens_cache_read) {
     rows.push({
@@ -71,7 +88,13 @@ export const ModelUsagePanel: React.FC<ModelUsageProps> = ({ usage }) => {
       {rows.map((row, idx) => {
         if (row.label === "---") {
           return (
-            <div key={`$usage-sep-${idx}`} className={styles.separator}></div>
+            <div
+              key={`$usage-sep-${idx}`}
+              className={clsx(
+                styles.separator,
+                row.padded ? styles.padded : undefined,
+              )}
+            ></div>
           );
         } else {
           return (

inspect_ai/_view/www/src/usage/TokenTable.tsx

@@ -1,17 +1,15 @@
 import clsx from "clsx";
+import { FC, ReactNode } from "react";
 import { ModelUsage1 } from "../types/log";
 import { ModelUsagePanel } from "./ModelUsagePanel";
 import styles from "./TokenTable.module.css";
 
 interface TokenTableProps {
   className?: string | string[];
-  children?: React.ReactNode;
+  children?: ReactNode;
 }
 
-export const TokenTable: React.FC<TokenTableProps> = ({
-  className,
-  children,
-}) => {
+export const TokenTable: FC<TokenTableProps> = ({ className, children }) => {
   return (
     <table
       className={clsx(
@@ -77,7 +75,7 @@ interface TokenRowProps {
   usage: ModelUsage1;
 }
 
-export const TokenRow: React.FC<TokenRowProps> = ({ model, usage }) => {
+export const TokenRow: FC<TokenRowProps> = ({ model, usage }) => {
   return (
     <tr>
       <td>