inspect-ai 0.3.89__py3-none-any.whl → 0.3.91__py3-none-any.whl

inspect_ai/_view/www/src/{types → @types}/log.d.ts

@@ -15,12 +15,14 @@ export type TaskVersion = number;
 export type TaskFile = string | null;
 export type TaskRegistryName = string | null;
 export type Solver = string | null;
-export type SolverArgs = {} | null;
+export type SolverArgs = {
+  [k: string]: unknown;
+} | null;
 export type Tags = string[] | null;
 export type Name = string | null;
 export type Location = string | null;
 export type Samples = number | null;
-export type SampleIds = (number | string)[] | null;
+export type SampleIds = string[] | number[] | (string | number)[] | null;
 export type Shuffled = boolean | null;
 export type Type = string;
 export type Model = string;
@@ -49,11 +51,13 @@ export type MaxToolOutput = number | null;
 export type CachePrompt = "auto" | boolean | null;
 export type ReasoningEffort = ("low" | "medium" | "high") | null;
 export type ReasoningTokens = number | null;
+export type ReasoningSummary = ("concise" | "detailed" | "auto") | null;
 export type ReasoningHistory = ("none" | "all" | "last" | "auto") | null;
 export type Name1 = string;
 export type Type1 =
   | ("string" | "integer" | "number" | "boolean" | "array" | "object" | "null")
   | null;
+export type Format = string | null;
 export type Description = string | null;
 export type Enum = unknown[] | null;
 export type Properties = {
@@ -71,13 +75,20 @@ export type ModelRoles = {
 export type Model1 = string;
 export type BaseUrl = string | null;
 export type Limit = number | [unknown, unknown] | null;
-export type SampleId = string | number | (string | number)[] | null;
+export type SampleId =
+  | string
+  | number
+  | string[]
+  | number[]
+  | (string | number)[]
+  | null;
 export type Epochs = number | null;
 export type EpochsReducer = string[] | null;
 export type Name2 = string;
 export type Tools = string | string[];
 export type Approvers = ApproverPolicyConfig[];
 export type FailOnError = boolean | number | null;
+export type RetryOnError = number | null;
 export type MessageLimit = number | null;
 export type TokenLimit = number | null;
 export type TimeLimit = number | null;
@@ -95,10 +106,14 @@ export type ScoreDisplay = boolean | null;
 export type Type2 = "git";
 export type Origin = string;
 export type Commit = string;
-export type Metadata = {} | null;
+export type Metadata = {
+  [k: string]: unknown;
+} | null;
 export type Scorers = EvalScorer[] | null;
 export type Name3 = string;
-export type Options = {} | null;
+export type Options = {
+  [k: string]: unknown;
+} | null;
 export type Metrics =
   | (
       | EvalMetricDefinition
@@ -111,8 +126,12 @@ export type Metrics =
     }
   | null;
 export type Name4 = string;
-export type Options1 = {} | null;
-export type Metadata1 = {} | null;
+export type Options1 = {
+  [k: string]: unknown;
+} | null;
+export type Metadata1 = {
+  [k: string]: unknown;
+} | null;
 export type Metrics1 =
   | EvalMetricDefinition[]
   | {
@@ -129,10 +148,16 @@ export type Scorer = string;
 export type Reducer = string | null;
 export type Name7 = string;
 export type Value = number;
-export type Metadata2 = {} | null;
-export type Metadata3 = {} | null;
+export type Metadata2 = {
+  [k: string]: unknown;
+} | null;
+export type Metadata3 = {
+  [k: string]: unknown;
+} | null;
 export type Scores = EvalScore[];
-export type Metadata4 = {} | null;
+export type Metadata4 = {
+  [k: string]: unknown;
+} | null;
 export type StartedAt = string;
 export type CompletedAt = string;
 export type InputTokens = number;
@@ -177,10 +202,10 @@ export type Image = string;
 export type Detail = "auto" | "low" | "high";
 export type Type6 = "audio";
 export type Audio = string;
-export type Format = "wav" | "mp3";
+export type Format1 = "wav" | "mp3";
 export type Type7 = "video";
 export type Video = string;
-export type Format1 = "mp4" | "mpeg" | "mov";
+export type Format2 = "mp4" | "mpeg" | "mov";
 export type Source = ("input" | "generate") | null;
 export type Role = "system";
 export type Id2 = string | null;
@@ -213,7 +238,7 @@ export type Id4 = string;
 export type Function = string;
 export type ParseError = string | null;
 export type Title = string | null;
-export type Format2 = "text" | "markdown";
+export type Format3 = "text" | "markdown";
 export type Content3 = string;
 export type Type8 = string | null;
 export type Model2 = string | null;
@@ -270,7 +295,9 @@ export type Bytes1 = number[] | null;
 export type Content5 = Logprob[];
 export type Choices1 = ChatCompletionChoice[];
 export type Time = number | null;
-export type Metadata5 = {} | null;
+export type Metadata5 = {
+  [k: string]: unknown;
+} | null;
 export type Error = string | null;
 export type Scores1 = {
   [k: string]: Score;
@@ -285,7 +312,9 @@ export type Value1 =
     };
 export type Answer = string | null;
 export type Explanation = string | null;
-export type Metadata6 = {} | null;
+export type Metadata6 = {
+  [k: string]: unknown;
+} | null;
 export type Timestamp = string;
 export type WorkingStart = number;
 export type Pending = boolean | null;
@@ -301,7 +330,9 @@ export type Input1 =
 export type Choices2 = string[] | null;
 export type Target1 = string | string[];
 export type Id6 = number | string | null;
-export type Metadata8 = {} | null;
+export type Metadata8 = {
+  [k: string]: unknown;
+} | null;
 export type Files1 = {
   [k: string]: string;
 } | null;
@@ -522,6 +553,7 @@ export type Events = (
 export type TotalTime = number | null;
 export type WorkingTime3 = number | null;
 export type Uuid = string | null;
+export type ErrorRetries = EvalError[] | null;
 export type Type15 =
   | "context"
   | "time"
@@ -544,7 +576,9 @@ export type Value2 =
     };
 export type Answer1 = string | null;
 export type Explanation2 = string | null;
-export type Metadata9 = {} | null;
+export type Metadata9 = {
+  [k: string]: unknown;
+} | null;
 export type SampleId1 = string | number | null;
 export type Samples2 = EvalSampleScore[];
 export type Location1 = string;
@@ -594,8 +628,12 @@ export interface EvalSpec {
   scorers: Scorers;
   metrics: Metrics1;
 }
-export interface TaskAttribs {}
-export interface TaskArgs {}
+export interface TaskAttribs {
+  [k: string]: unknown;
+}
+export interface TaskArgs {
+  [k: string]: unknown;
+}
 /**
  * Dataset used for evaluation.
  */
@@ -643,6 +681,7 @@ export interface GenerateConfig {
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
   reasoning_tokens: ReasoningTokens;
+  reasoning_summary: ReasoningSummary;
   reasoning_history: ReasoningHistory;
   response_schema: ResponseSchema | null;
 }
@@ -660,6 +699,7 @@ export interface ResponseSchema {
  */
 export interface JSONSchema {
   type: Type1;
+  format: Format;
   description: Description;
   default: Default;
   enum: Enum;
@@ -672,7 +712,9 @@ export interface JSONSchema {
 export interface Default {
   [k: string]: unknown;
 }
-export interface ModelArgs {}
+export interface ModelArgs {
+  [k: string]: unknown;
+}
 /**
  * Model config.
  */
@@ -682,7 +724,9 @@ export interface EvalModelConfig {
   base_url: BaseUrl;
   args: Args;
 }
-export interface Args {}
+export interface Args {
+  [k: string]: unknown;
+}
 /**
  * Configuration used for evaluation.
  */
@@ -693,6 +737,7 @@ export interface EvalConfig {
   epochs_reducer: EpochsReducer;
   approval: ApprovalPolicyConfig | null;
   fail_on_error: FailOnError;
+  retry_on_error: RetryOnError;
   message_limit: MessageLimit;
   token_limit: TokenLimit;
   time_limit: TimeLimit;
@@ -732,7 +777,9 @@ export interface ApproverPolicyConfig {
   tools: Tools;
   params: Params;
 }
-export interface Params {}
+export interface Params {
+  [k: string]: unknown;
+}
 /**
  * Git revision for evaluation.
  */
@@ -770,7 +817,9 @@ export interface EvalPlanStep {
   solver: Solver1;
   params: Params1;
 }
-export interface Params1 {}
+export interface Params1 {
+  [k: string]: unknown;
+}
 /**
  * Model generation options.
  */
@@ -798,6 +847,7 @@ export interface GenerateConfig1 {
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
   reasoning_tokens: ReasoningTokens;
+  reasoning_summary: ReasoningSummary;
   reasoning_history: ReasoningHistory;
   response_schema: ResponseSchema | null;
 }
@@ -821,7 +871,9 @@ export interface EvalScore {
   metrics: Metrics2;
   metadata: Metadata3;
 }
-export interface Params2 {}
+export interface Params2 {
+  [k: string]: unknown;
+}
 export interface Metrics2 {
   [k: string]: EvalMetric;
 }
@@ -834,7 +886,9 @@ export interface EvalMetric {
   params: Params3;
   metadata: Metadata2;
 }
-export interface Params3 {}
+export interface Params3 {
+  [k: string]: unknown;
+}
 /**
  * Timing and usage statistics.
  */
@@ -888,6 +942,7 @@ export interface EvalSample {
   working_time: WorkingTime3;
   uuid: Uuid;
   error: EvalError | null;
+  error_retries: ErrorRetries;
   attachments: Attachments;
   limit: EvalSampleLimit | null;
 }
@@ -934,7 +989,7 @@ export interface ContentImage {
 export interface ContentAudio {
   type: Type6;
   audio: Audio;
-  format: Format;
+  format: Format1;
 }
 /**
  * Video content.
@@ -942,7 +997,7 @@ export interface ContentAudio {
 export interface ContentVideo {
   type: Type7;
   video: Video;
-  format: Format1;
+  format: Format2;
 }
 /**
  * User chat message.
@@ -976,13 +1031,15 @@ export interface ToolCall {
   view: ToolCallContent | null;
   type: Type8;
 }
-export interface Arguments {}
+export interface Arguments {
+  [k: string]: unknown;
+}
 /**
  * Content to include in tool call view.
  */
 export interface ToolCallContent {
   title: Title;
-  format: Format2;
+  format: Format3;
   content: Content3;
 }
 /**
@@ -1053,8 +1110,12 @@ export interface Score {
   explanation: Explanation;
   metadata: Metadata6;
 }
-export interface Metadata7 {}
-export interface Store {}
+export interface Metadata7 {
+  [k: string]: unknown;
+}
+export interface Store {
+  [k: string]: unknown;
+}
 /**
  * Beginning of processing a Sample.
  */
@@ -1372,7 +1433,9 @@ export interface SubtaskEvent {
   completed: Completed2;
   working_time: WorkingTime1;
 }
-export interface Input5 {}
+export interface Input5 {
+  [k: string]: unknown;
+}
 export interface Result2 {
   [k: string]: unknown;
 }

inspect_ai/_view/www/{App.css → src/app/App.css}

@@ -83,39 +83,39 @@ body[class^="vscode-"] .app-main-grid {
 
 /* Inspect Font Size Styles */
 .text-size-title {
-  font-size: var(--inspect-font-size-title);
+  font-size: var(--inspect-font-size-title) !important;
 }
 
 .text-size-title-secondary {
-  font-size: var(--inspect-font-size-title-secondary);
+  font-size: var(--inspect-font-size-title-secondary) !important;
 }
 
 .text-size-largest {
-  font-size: var(--inspect-font-size-largest);
+  font-size: var(--inspect-font-size-largest) !important;
 }
 
 .text-size-larger {
-  font-size: var(--inspect-font-size-larger);
+  font-size: var(--inspect-font-size-larger) !important;
 }
 
 .text-size-large {
-  font-size: var(--inspect-font-size-large);
+  font-size: var(--inspect-font-size-large) !important;
 }
 
 .text-size-base {
-  font-size: var(--inspect-font-size-base);
+  font-size: var(--inspect-font-size-base) !important;
 }
 
 .text-size-small {
-  font-size: var(--inspect-font-size-small);
+  font-size: var(--inspect-font-size-small) !important;
 }
 
 .text-size-smaller {
-  font-size: var(--inspect-font-size-smaller);
+  font-size: var(--inspect-font-size-smaller) !important;
 }
 
 .text-size-smallest {
-  font-size: var(--inspect-font-size-smallest);
+  font-size: var(--inspect-font-size-smallest) !important;
 }
 
 .text-truncate {
@@ -456,8 +456,8 @@ body {
 
 .markdown-content pre > code,
 .markdown-content pre {
-  white-space: pre-wrap;
-  word-wrap: anywhere;
+  white-space: pre-wrap !important;
+  word-wrap: anywhere !important;
 }
 
 .log pre code {
@@ -493,9 +493,7 @@ body {
   padding-top: 10em;
 }
 
-code:not(.sourceCode) {
-  border-radius: 2px;
-  padding: 0.2em;
+code:not(.sourceCode):not(.source-code) {
   color: var(--bs-code-color);
 }
 
@@ -1023,6 +1021,16 @@ ul.jsondiffpatch-textdiff {
   font-weight: bold;
 }
 
+/* prism-custom.css */
+code[class*="language-"],
+pre[class*="language-"] {
+  font-size: 0.8rem !important;
+}
+
+.token {
+  font-size: 0.8rem !important;
+}
+
 /* PrismJS 1.29.0 https://prismjs.com/download.html#themes=prism-dark&languages=markup+css+clike+javascript+bash+python */
 /* This has been generated from the URL above, then scoped within the
  * .vscode-dark class. If it needs to be regenerated, be sure to add that back in. */

inspect_ai/_view/www/src/app/App.tsx

@@ -0,0 +1,168 @@
+import "bootstrap-icons/font/bootstrap-icons.css";
+import "bootstrap/dist/css/bootstrap.css";
+import JSON5 from "json5";
+
+import "prismjs";
+import "prismjs/components/prism-bash";
+import "prismjs/components/prism-clike";
+import "prismjs/components/prism-javascript";
+import "prismjs/components/prism-json";
+import "prismjs/components/prism-python";
+import "prismjs/themes/prism.css";
+
+import "./App.css";
+
+import ClipboardJS from "clipboard";
+import { FC, useCallback, useEffect } from "react";
+import { RouterProvider } from "react-router-dom";
+import { ClientAPI, HostMessage } from "../client/api/types.ts";
+import { useStore } from "../state/store.ts";
+import { AppRouter } from "./routing/AppRouter.tsx";
+
+interface AppProps {
+  api: ClientAPI;
+}
+
+/**
+ * Renders the Main Application
+ */
+export const App: FC<AppProps> = ({ api }) => {
+  const setAppStatus = useStore((state) => state.appActions.setStatus);
+  const setLogs = useStore((state) => state.logsActions.setLogs);
+  const selectLogFile = useStore((state) => state.logsActions.selectLogFile);
+  const setIntialState = useStore((state) => state.appActions.setInitialState);
+  const rehydrated = useStore((state) => state.app.rehydrated);
+  const refreshLogs = useStore((state) => state.logsActions.refreshLogs);
+  const loadLog = useStore((state) => state.logActions.loadLog);
+  const pollLog = useStore((state) => state.logActions.pollLog);
+  const loadedLogFile = useStore((state) => state.log.loadedLog);
+  const selectedLogFile = useStore((state) =>
+    state.logsActions.getSelectedLogFile(),
+  );
+  const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
+  const logs = useStore((state) => state.logs.logs);
+
+  // Load a specific log
+  useEffect(() => {
+    const loadSpecificLog = async () => {
+      if (selectedLogFile && selectedLogFile !== loadedLogFile) {
+        try {
+          // Set loading first and wait for it to update
+          setAppStatus({ loading: true, error: undefined });
+
+          // Then load the log
+          await loadLog(selectedLogFile);
+
+          // Finally set loading to false
+          setAppStatus({ loading: false, error: undefined });
+        } catch (e) {
+          console.log(e);
+          setAppStatus({ loading: false, error: e as Error });
+        }
+      }
+    };
+
+    loadSpecificLog();
+  }, [selectedLogFile, loadedLogFile, loadLog, setAppStatus]);
+
+  useEffect(() => {
+    // If the component re-mounts and there is a running load loaded
+    // start up polling
+    const doPoll = async () => {
+      await pollLog();
+    };
+    if (selectedLogSummary?.status === "started") {
+      doPoll();
+    }
+  }, [pollLog, selectedLogSummary?.status]);
+
+  useEffect(() => {
+    if (logs.log_dir && logs.files.length === 0) {
+      setAppStatus({
+        loading: false,
+        error: new Error(
+          `No log files to display in the directory ${logs.log_dir}. Are you sure this is the correct log directory?`,
+        ),
+      });
+    }
+  }, [logs.log_dir, logs.files.length, setAppStatus]);
+
+  const onMessage = useCallback(
+    async (e: HostMessage) => {
+      switch (e.data.type) {
+        case "updateState": {
+          if (e.data.url) {
+            const decodedUrl = decodeURIComponent(e.data.url);
+            setIntialState(decodedUrl, e.data.sample_id, e.data.sample_epoch);
+          }
+          break;
+        }
+        case "backgroundUpdate": {
+          const decodedUrl = decodeURIComponent(e.data.url);
+          const log_dir = e.data.log_dir;
+          const isFocused = document.hasFocus();
+          if (!isFocused) {
+            if (log_dir === logs.log_dir) {
+              selectLogFile(decodedUrl);
+            } else {
+              api.open_log_file(e.data.url, e.data.log_dir);
+            }
+          } else {
+            refreshLogs();
+          }
+          break;
+        }
+      }
+    },
+    [logs, selectLogFile, refreshLogs, api],
+  );
+
+  // listen for updateState messages from vscode
+  useEffect(() => {
+    window.addEventListener("message", onMessage);
+    return () => {
+      window.removeEventListener("message", onMessage);
+    };
+  }, [onMessage]);
+
+  useEffect(() => {
+    const loadLogsAndState = async () => {
+      // First see if there is embedded state and if so, use that
+      const embeddedState = document.getElementById("logview-state");
+      if (embeddedState && !rehydrated) {
+        const state = JSON5.parse(embeddedState.textContent || "");
+        onMessage({ data: state });
+      } else {
+        // For non-route URL params support (legacy)
+        const urlParams = new URLSearchParams(window.location.search);
+
+        // If the URL provides a task file, load that
+        const logPath = urlParams.get("task_file");
+
+        // Replace spaces with a '+' sign:
+        const resolvedLogPath = logPath ? logPath.replace(" ", "+") : logPath;
+
+        if (resolvedLogPath) {
+          // Load only this file
+          setLogs({
+            log_dir: "",
+            files: [{ name: resolvedLogPath }],
+          });
+        } else {
+          // If a log file was passed, select it
+          const log_file = urlParams.get("log_file");
+          if (log_file) {
+            await selectLogFile(log_file);
+          }
+          // Else do nothing - RouteProvider will handle it
+        }
+      }
+
+      new ClipboardJS(".clipboard-button,.copy-button");
+    };
+
+    loadLogsAndState();
+  }, [setLogs, selectLogFile, refreshLogs, onMessage]);
+
+  return <RouterProvider router={AppRouter} />;
+};

inspect_ai/_view/www/src/{AppErrorBoundary.tsx → app/AppErrorBoundary.tsx}

@@ -1,5 +1,5 @@
 import { Component, ErrorInfo, ReactNode } from "react";
-import { ErrorPanel } from "./components/ErrorPanel";
+import { ErrorPanel } from "../components/ErrorPanel";
 
 interface Props {
   children: ReactNode;

inspect_ai/_view/www/src/{appearance → app/appearance}/icons.ts

@@ -76,6 +76,7 @@ export const ApplicationIcons = {
   play: "bi bi-play-fill",
   previous: "bi bi-chevron-left",
   refresh: "bi bi-arrow-clockwise",
+  retry: "bi bi-arrow-repeat",
   role: {
     user: "bi bi-person",
     system: "bi bi-cpu",

inspect_ai/_view/www/src/{metadata → app/content}/RenderedContent.tsx

@@ -1,15 +1,15 @@
 import JSON5 from "json5";
 import { ApplicationIcons } from "../appearance/icons";
 
-import { ANSIDisplay } from "../components/AnsiDisplay";
-import { ChatMessageRenderer } from "../samples/chat/ChatMessageRenderer";
-import { formatNumber } from "../utils/format";
+import { ChatMessageRenderer } from "../../app/samples/chat/ChatMessageRenderer";
+import { ANSIDisplay } from "../../components/AnsiDisplay";
+import { formatNumber } from "../../utils/format";
 import { MetaDataView } from "./MetaDataView";
 
 import clsx from "clsx";
 import { FC, Fragment, isValidElement, JSX, ReactNode } from "react";
-import JSONPanel from "../components/JsonPanel";
-import { isJson } from "../utils/json";
+import JSONPanel from "../../components/JsonPanel";
+import { isJson } from "../../utils/json";
 import styles from "./RenderedContent.module.css";
 import { Buckets, ContentRenderer } from "./types";