inspect-ai 0.3.61__py3-none-any.whl → 0.3.63__py3-none-any.whl

inspect_ai/_view/www/src/samples/transcript/state/{StateEventView.mjs → StateEventView.tsx}

@@ -1,33 +1,43 @@
 // @ts-check
-import { html } from "htm/preact";
 
-import { EventPanel } from "../EventPanel.mjs";
+import clsx from "clsx";
+import {
+  Changes,
+  JsonChange,
+  Op,
+  StateEvent,
+  StoreEvent,
+} from "../../../types/log";
+import { formatDateTime } from "../../../utils/format";
+import { EventPanel } from "../event/EventPanel";
+import { TranscriptEventState } from "../types";
+import { StateDiffView } from "./StateDiffView";
 import {
   RenderableChangeTypes,
   StoreSpecificRenderableTypes,
-} from "./StateEventRenderers.mjs";
-import { StateDiffView } from "./StateDiffView.mjs";
-import { formatDateTime } from "../../../utils/Format.mjs";
+} from "./StateEventRenderers";
+
+import styles from "./StateEventView.module.css";
+
+interface StateEventViewProps {
+  id: string;
+  event: StateEvent | StoreEvent;
+  eventState: TranscriptEventState;
+  setEventState: (state: TranscriptEventState) => void;
+  isStore?: boolean;
+  className?: string | string[];
+}
 
 /**
  * Renders the StateEventView component.
- *
- * @param {Object} props - The properties passed to the component.
- * @param { string  } props.id - The id of this event.
- * @param {import("../../../types/log").StateEvent } props.event - The event object to display.
- * @param {import("./../Types.mjs").TranscriptEventState} props.eventState - The state for this event
- * @param {(state: import("./../Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
- * @param { boolean } props.isStore - Whether this event view is rendering a storage (rather than a state)
- * @param { Object } props.style - The style of this event.
- * @returns {import("preact").JSX.Element} The component.
  */
-export const StateEventView = ({
+export const StateEventView: React.FC<StateEventViewProps> = ({
   id,
   event,
   eventState,
   setEventState,
-  isStore,
-  style,
+  isStore = false,
+  className,
 }) => {
   const summary = summarizeChanges(event.changes);
 
@@ -35,14 +45,14 @@ export const StateEventView = ({
   const [before, after] = synthesizeComparable(event.changes);
 
   const tabs = [
-    html`<${StateDiffView}
-      before=${before}
-      after=${after}
-      name="Diff"
-      style=${{ margin: "1em 0em" }}
-    />`,
+    <StateDiffView
+      before={before}
+      after={after}
+      data-name="Diff"
+      className={clsx(styles.diff)}
+    />,
   ];
-  // This clone is important since the state is used by preact as potential values that are rendered
+  // This clone is important since the state is used by react as potential values that are rendered
   // and as a result may be decorated with additional properties, etc..., resulting in DOM elements
   // appearing attached to state.
   const changePreview = generatePreview(
@@ -52,45 +62,45 @@ export const StateEventView = ({
   );
   if (changePreview) {
     tabs.unshift(
-      html`<div name="Summary" style=${{ margin: "1em 0em", width: "100%" }}>
-        ${changePreview}
-      </div>`,
+      <div data-name="Summary" className={clsx(styles.summary)}>
+        {changePreview}
+      </div>,
     );
   }
 
   // Compute the title
   const title = event.event === "state" ? "State Updated" : "Store Updated";
 
-  return html`
-  <${EventPanel} 
-    id=${id} 
-    title="${title}" 
-    subTitle=${formatDateTime(new Date(event.timestamp))} 
-    text=${tabs.length === 1 ? summary : undefined} 
-    collapse=${changePreview === undefined ? true : undefined} 
-    style=${style}
-    selectedNav=${eventState.selectedNav || ""}
-    onSelectedNav=${(selectedNav) => {
-      setEventState({ ...eventState, selectedNav });
-    }}
-    collapsed=${eventState.collapsed}
-    onCollapsed=${(collapsed) => {
-      setEventState({ ...eventState, collapsed });
-    }}                  
-  >
-    ${tabs}
-  </${EventPanel}>`;
+  return (
+    <EventPanel
+      id={id}
+      title={title}
+      className={className}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      text={tabs.length === 1 ? summary : undefined}
+      collapse={changePreview === undefined ? true : undefined}
+      selectedNav={eventState.selectedNav || ""}
+      setSelectedNav={(selectedNav) => {
+        setEventState({ ...eventState, selectedNav });
+      }}
+      collapsed={eventState.collapsed}
+      setCollapsed={(collapsed) => {
+        setEventState({ ...eventState, collapsed });
+      }}
+    >
+      {tabs}
+    </EventPanel>
+  );
 };
 
 /**
  * Renders the value of a change based on its type.
- *
- * @param {import("../../../types/log").JsonChange[]} changes - The change object containing the value.
- * @param {Object} resolvedState - The change object containing the value.
- * @param {boolean} isStore - Is this rendering a store event
- * @returns {import("preact").JSX.Element|Object|string|undefined} - The rendered HTML template if the value is an object with content and source, otherwise the value itself.
  */
-const generatePreview = (changes, resolvedState, isStore) => {
+const generatePreview = (
+  changes: JsonChange[],
+  resolvedState: Record<string, unknown>,
+  isStore: boolean,
+) => {
   const results = [];
   for (const changeType of [
     ...RenderableChangeTypes,
@@ -105,15 +115,41 @@ const generatePreview = (changes, resolvedState, isStore) => {
       changeType.signature.add.length;
     let matchingOps = 0;
     for (const change of changes) {
-      if (
-        changeType.signature[change.op] &&
-        changeType.signature[change.op].length > 0
-      ) {
-        changeType.signature[change.op].forEach((signature) => {
-          if (change.path.match(signature)) {
-            matchingOps++;
+      const op = change.op;
+      switch (op) {
+        case "add":
+          if (changeType.signature.add && changeType.signature.add.length > 0) {
+            changeType.signature.add.forEach((signature) => {
+              if (change.path.match(signature)) {
+                matchingOps++;
+              }
+            });
+          }
+          break;
+        case "remove":
+          if (
+            changeType.signature.remove &&
+            changeType.signature.remove.length > 0
+          ) {
+            changeType.signature.remove.forEach((signature) => {
+              if (change.path.match(signature)) {
+                matchingOps++;
+              }
+            });
           }
-        });
+          break;
+        case "replace":
+          if (
+            changeType.signature.replace &&
+            changeType.signature.replace.length > 0
+          ) {
+            changeType.signature.replace.forEach((signature) => {
+              if (change.path.match(signature)) {
+                matchingOps++;
+              }
+            });
+          }
+          break;
       }
     }
     if (matchingOps === requiredMatchCount) {
@@ -129,12 +165,9 @@ const generatePreview = (changes, resolvedState, isStore) => {
 
 /**
  * Renders the value of a change based on its type.
- *
- * @param {import("../../../types/log").JsonChange[]} changes - The change object containing the value.
- * @returns {string} - A string summarizing the changes
  */
-const summarizeChanges = (changes) => {
-  const changeMap = {
+const summarizeChanges = (changes: JsonChange[]): string => {
+  const changeMap: Record<Op, string[]> = {
     add: [],
     copy: [],
     move: [],
@@ -143,24 +176,43 @@ const summarizeChanges = (changes) => {
     test: [],
   };
   for (const change of changes) {
-    changeMap[change.op].push(change.path);
+    switch (change.op) {
+      case "add":
+        changeMap.add.push(change.path);
+        break;
+      case "copy":
+        changeMap.copy.push(change.path);
+        break;
+      case "move":
+        changeMap.move.push(change.path);
+        break;
+      case "replace":
+        changeMap.replace.push(change.path);
+        break;
+      case "remove":
+        changeMap.remove.push(change.path);
+        break;
+      case "test":
+        changeMap.test.push(change.path);
+        break;
+    }
   }
 
-  const changeList = [];
+  const changeList: string[] = [];
   const totalOpCount = Object.keys(changeMap).reduce((prev, current) => {
-    return prev + changeMap[current].length;
+    return prev + changeMap[current as Op].length;
   }, 0);
 
   if (totalOpCount > 2) {
     Object.keys(changeMap).forEach((key) => {
-      const opChanges = changeMap[key];
+      const opChanges = changeMap[key as Op];
       if (opChanges.length > 0) {
         changeList.push(`${key} ${opChanges.length}`);
       }
     });
   } else {
     Object.keys(changeMap).forEach((key) => {
-      const opChanges = changeMap[key];
+      const opChanges = changeMap[key as Op];
       if (opChanges.length > 0) {
         changeList.push(`${key} ${opChanges.join(", ")}`);
       }
@@ -171,11 +223,8 @@ const summarizeChanges = (changes) => {
 
 /**
  * Renders a view displaying a list of state changes.
- *
- * @param {import("../../../types/log").Changes} changes - The list of changes to be displayed.
- * @returns {[Object, Object]} The before and after objects
  */
-const synthesizeComparable = (changes) => {
+const synthesizeComparable = (changes: Changes) => {
   const before = {};
   const after = {};
 
@@ -192,7 +241,7 @@ const synthesizeComparable = (changes) => {
         setPath(after, change.path, change.value);
         break;
       case "move":
-        setPath(before, change.from, change.value);
+        setPath(before, change.from || "", change.value);
         setPath(after, change.path, change.value);
         break;
       case "remove":
@@ -211,15 +260,14 @@ const synthesizeComparable = (changes) => {
 
 /**
  * Sets a value at a path in an object
- *
- * @param {Object} target - The object into which to set the path
- * @param {string} path - The path of the value to set
- * @param {unknown} value - The value to set
- * @returns {Object} The mutated object
  */
-function setPath(target, path, value) {
+function setPath(
+  target: Record<string, unknown>,
+  path: string,
+  value: unknown,
+): void {
   const keys = parsePath(path);
-  let current = target;
+  let current: Record<string, unknown> = target;
 
   for (let i = 0; i < keys.length - 1; i++) {
     const key = keys[i];
@@ -227,7 +275,7 @@ function setPath(target, path, value) {
       // If the next key is a number, create an array, otherwise an object
       current[key] = isArrayIndex(keys[i + 1]) ? [] : {};
     }
-    current = current[key];
+    current = current[key] as Record<string, unknown>;
   }
 
   const lastKey = keys[keys.length - 1];
@@ -236,62 +284,55 @@ function setPath(target, path, value) {
 
 /**
  * Places structure in an object (without placing values)
- *
- * @param {Object} target - The object into which to initialize the path
- * @param {string} path - The path of the value to set
- * @returns {Object} The mutated object
  */
-function initializeArrays(target, path) {
+function initializeArrays(target: Record<string, unknown>, path: string): void {
   const keys = parsePath(path);
-  let current = target;
+  let current: Record<string, unknown> = target;
 
   for (let i = 0; i < keys.length - 1; i++) {
     const key = keys[i];
     const nextKey = keys[i + 1];
 
     if (isArrayIndex(nextKey)) {
-      current[key] = initializeArray(current[key], nextKey);
+      current[key] = initializeArray(
+        current[key] as string[] | undefined,
+        nextKey,
+      );
     } else {
-      current[key] = initializeObject(current[key]);
+      current[key] = initializeObject(current[key] as object | undefined);
     }
 
-    current = current[key];
+    current = current[key] as Record<string, unknown>;
   }
 
   const lastKey = keys[keys.length - 1];
   if (isArrayIndex(lastKey)) {
-    initializeArray(current, lastKey);
+    const lastValue = current[lastKey] as string[] | undefined;
+    initializeArray(lastValue, lastKey);
   }
 }
 
 /**
  * Parses a path into an array of keys
- *
- * @param {string} path - The path to split
- * @returns {string[]} Array of keys
  */
-function parsePath(path) {
+function parsePath(path: string): string[] {
   return path.split("/").filter(Boolean);
 }
 
 /**
  * Checks if a key represents an array index
- *
- * @param {string} key - The key to check
- * @returns {boolean} True if the key is a number
  */
-function isArrayIndex(key) {
+function isArrayIndex(key: string): boolean {
   return /^\d+$/.test(key);
 }
 
 /**
  * Initializes an array at a given key, ensuring it is large enough
- *
- * @param {Array|undefined} current - The current array or undefined
- * @param {string} nextKey - The key of the next array index
- * @returns {Array} Initialized array
  */
-function initializeArray(current, nextKey) {
+function initializeArray(
+  current: Array<string> | undefined,
+  nextKey: string,
+): Array<string> {
   if (!Array.isArray(current)) {
     current = [];
   }
@@ -304,10 +345,7 @@ function initializeArray(current, nextKey) {
 
 /**
  * Initializes an object at a given key if it doesn't exist
- *
- * @param {Object|undefined} current - The current object or undefined
- * @returns {Object} Initialized object
  */
-function initializeObject(current) {
+function initializeObject(current?: Object): Object {
   return current ?? {};
 }

inspect_ai/_view/www/src/samples/transcript/types.ts

@@ -0,0 +1,58 @@
+import {
+  ApprovalEvent,
+  Changes,
+  ErrorEvent,
+  InfoEvent,
+  InputEvent,
+  LoggerEvent,
+  ModelEvent,
+  SampleInitEvent,
+  SampleLimitEvent,
+  ScoreEvent,
+  StateEvent,
+  StepEvent,
+  StoreEvent,
+  SubtaskEvent,
+  ToolEvent,
+} from "../../types/log";
+
+export interface StateManager {
+  scope: string;
+  getState(): object;
+  initializeState(state: object): void;
+  applyChanges(changes: Changes): object;
+}
+
+export type EventType =
+  | SampleInitEvent
+  | SampleLimitEvent
+  | StateEvent
+  | StoreEvent
+  | ModelEvent
+  | LoggerEvent
+  | InfoEvent
+  | StepEvent
+  | SubtaskEvent
+  | ScoreEvent
+  | ToolEvent
+  | InputEvent
+  | ErrorEvent
+  | ApprovalEvent;
+
+export class EventNode {
+  event: EventType;
+  children: EventNode[] = [];
+  depth: number;
+
+  constructor(event: EventType, depth: number) {
+    this.event = event;
+    this.depth = depth;
+  }
+}
+
+export interface TranscriptEventState {
+  selectedNav?: string;
+  collapsed?: boolean;
+}
+
+export type TranscriptState = Record<string, TranscriptEventState>;

inspect_ai/_view/www/src/types/log.d.ts

@@ -70,7 +70,6 @@ export type LogitBias = {
   [k: string]: number;
 } | null;
 export type Seed = number | null;
-export type Suffix = string | null;
 export type TopK = number | null;
 export type NumChoices = number | null;
 export type Logprobs = boolean | null;
@@ -80,6 +79,7 @@ export type InternalTools = boolean | null;
 export type MaxToolOutput = number | null;
 export type CachePrompt = "auto" | boolean | null;
 export type ReasoningEffort = ("low" | "medium" | "high") | null;
+export type ReasoningHistory = boolean | null;
 export type TotalSamples = number;
 export type CompletedSamples = number;
 export type Name3 = string;
@@ -133,7 +133,7 @@ export type Content1 =
   | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source1 = ("input" | "generate") | null;
 export type Role1 = "user";
-export type ToolCallId = string | null;
+export type ToolCallId = string[] | null;
 export type Content2 =
   | string
   | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
@@ -147,6 +147,7 @@ export type ParseError = string | null;
 export type Title = string | null;
 export type Format2 = "text" | "markdown";
 export type Content3 = string;
+export type Reasoning = string | null;
 export type Content4 =
   | string
   | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
@@ -323,6 +324,7 @@ export type Event10 = "logger";
 export type Name7 = string | null;
 export type Level =
   | "debug"
+  | "trace"
   | "http"
   | "sandbox"
   | "info"
@@ -547,7 +549,6 @@ export interface GenerateConfig {
   presence_penalty: PresencePenalty;
   logit_bias: LogitBias;
   seed: Seed;
-  suffix: Suffix;
   top_k: TopK;
   num_choices: NumChoices;
   logprobs: Logprobs;
@@ -557,6 +558,7 @@ export interface GenerateConfig {
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
+  reasoning_history: ReasoningHistory;
 }
 export interface EvalResults {
   total_samples: TotalSamples;
@@ -658,6 +660,7 @@ export interface ChatMessageAssistant {
   source: Source2;
   role: Role2;
   tool_calls: ToolCalls;
+  reasoning: Reasoning;
 }
 export interface ToolCall {
   id: Id1;
@@ -901,7 +904,6 @@ export interface GenerateConfig1 {
   presence_penalty: PresencePenalty;
   logit_bias: LogitBias;
   seed: Seed;
-  suffix: Suffix;
   top_k: TopK;
   num_choices: NumChoices;
   logprobs: Logprobs;
@@ -911,6 +913,7 @@ export interface GenerateConfig1 {
   max_tool_output: MaxToolOutput;
   cache_prompt: CachePrompt;
   reasoning_effort: ReasoningEffort;
+  reasoning_history: ReasoningHistory;
 }
 /**
  * Model call (raw request/response data).

inspect_ai/_view/www/src/types/prism.d.ts

@@ -1,4 +1,15 @@
 declare var Prism: {
   languages: any;
   highlight(contents: any, tokens: any, type: any): string;
+  highlightElement(
+    element: HTMLElement,
+    async?: boolean,
+    callback?: (element: HTMLElement) => void,
+  );
+
+  highlightAllUnder(
+    element: HTMLElement,
+    async?: boolean,
+    callback?: (element: HTMLElement) => void,
+  );
 };

inspect_ai/_view/www/src/types.ts

@@ -0,0 +1,71 @@
+import {
+  EvalLogHeader,
+  EvalSummary,
+  LogFiles,
+  SampleSummary,
+} from "./api/types";
+import { ContentImage, ContentText, EvalSample } from "./types/log";
+
+export interface ApplicationState {
+  logs?: LogFiles;
+  selectedLogIndex?: number;
+  logHeaders?: Record<string, EvalLogHeader>;
+  headersLoading?: boolean;
+  selectedLog?: CurrentLog;
+  selectedWorkspaceTab?: string;
+  selectedSampleIndex?: number;
+  selectedSample?: EvalSample;
+  sampleStatus?: "loading" | "ok" | "error";
+  sampleError?: Error;
+  selectedSampleTab?: string;
+  sampleScrollPosition?: number;
+  showingSampleDialog?: boolean;
+  status?: AppStatus;
+  offcanvas?: boolean;
+  showFind?: boolean;
+  filter?: ScoreFilter;
+  epoch?: string;
+  sort?: string;
+  scores?: ScoreLabel[];
+  score?: ScoreLabel;
+  filteredSamples?: SampleSummary[];
+  groupBy?: "none" | "epoch" | "sample";
+  groupByOrder?: "asc" | "desc";
+  workspaceTabScrollPosition?: Record<string, number>;
+}
+
+export interface AppStatus {
+  loading: boolean;
+  error?: Error;
+}
+
+export interface Capabilities {
+  downloadFiles: boolean;
+  webWorkers: boolean;
+}
+
+export interface CurrentLog {
+  name: string;
+  contents: EvalSummary;
+}
+
+export interface Logs {
+  log_dir: string;
+  files: string[];
+}
+
+export interface ScoreLabel {
+  name: string;
+  scorer: string;
+}
+
+export interface ScoreFilter {
+  value?: string;
+}
+
+export type SampleMode = "none" | "single" | "many";
+
+export interface ContentTool {
+  type: "tool";
+  content: (ContentImage | ContentText)[];
+}

inspect_ai/_view/www/src/usage/ModelTokenTable.tsx

@@ -0,0 +1,22 @@
+import { TokenHeader, TokenRow, TokenTable } from "./TokenTable";
+
+interface ModelTokenTable {
+  model_usage: any;
+  className?: string | string[];
+}
+
+export const ModelTokenTable: React.FC<ModelTokenTable> = ({
+  model_usage,
+  className,
+}) => {
+  return (
+    <TokenTable className={className}>
+      <TokenHeader />
+      <tbody>
+        {Object.keys(model_usage).map((key) => {
+          return <TokenRow model={key} usage={model_usage[key]} />;
+        })}
+      </tbody>
+    </TokenTable>
+  );
+};

inspect_ai/_view/www/src/usage/ModelUsagePanel.module.css

@@ -0,0 +1,24 @@
+.wrapper {
+  display: grid;
+  grid-template-columns: 0 auto auto;
+  column-gap: 1.5em;
+  row-gap: 0.2em;
+}
+
+.col2 {
+  grid-column: 2;
+}
+
+.col1_3 {
+  grid-column: 1/3;
+}
+
+.col3 {
+  grid-column: 3;
+}
+
+.separator {
+  grid-column: -1/1;
+  height: 1px;
+  background-color: var(--bs-light-border-subtle);
+}