inspect-ai 0.3.93__py3-none-any.whl → 0.3.94__py3-none-any.whl

inspect_ai/_view/www/src/@types/log.d.ts

@@ -316,6 +316,7 @@ export type Explanation = string | null;
 export type Metadata6 = {
   [k: string]: unknown;
 } | null;
+export type SpanId = string | null;
 export type Timestamp = string;
 export type WorkingStart = number;
 export type Pending = boolean | null;
@@ -339,6 +340,7 @@ export type Files1 = {
 } | null;
 export type Setup1 = string | null;
 export type JsonValue = unknown;
+export type SpanId1 = string | null;
 export type Timestamp1 = string;
 export type WorkingStart1 = number;
 export type Pending1 = boolean | null;
@@ -352,6 +354,7 @@ export type Type10 =
   | "custom";
 export type Message2 = string;
 export type Limit1 = number | null;
+export type SpanId2 = string | null;
 export type Timestamp2 = string;
 export type WorkingStart2 = number;
 export type Pending2 = boolean | null;
@@ -366,6 +369,7 @@ export type Input2 = string | null;
 export type Result = number | null;
 export type Output = string | null;
 export type Completed = string | null;
+export type SpanId3 = string | null;
 export type Timestamp3 = string;
 export type WorkingStart3 = number;
 export type Pending3 = boolean | null;
@@ -374,11 +378,13 @@ export type Op = "remove" | "add" | "replace" | "move" | "test" | "copy";
 export type Path = string;
 export type From = string | null;
 export type Changes = JsonChange[];
+export type SpanId4 = string | null;
 export type Timestamp4 = string;
 export type WorkingStart4 = number;
 export type Pending4 = boolean | null;
 export type Event4 = "store";
 export type Changes1 = JsonChange[];
+export type SpanId5 = string | null;
 export type Timestamp5 = string;
 export type WorkingStart5 = number;
 export type Pending5 = boolean | null;
@@ -399,11 +405,13 @@ export type Additionalproperties1 = boolean;
 export type Tools1 = ToolInfo[];
 export type ToolChoice = ("auto" | "any" | "none") | ToolFunction;
 export type Name9 = string;
+export type Retries = number | null;
 export type Error1 = string | null;
 export type Cache = ("read" | "write") | null;
 export type Time1 = number | null;
 export type Completed1 = string | null;
 export type WorkingTime = number | null;
+export type SpanId6 = string | null;
 export type Timestamp6 = string;
 export type WorkingStart6 = number;
 export type Pending6 = boolean | null;
@@ -428,6 +436,7 @@ export type Result1 =
       | ContentVideo
     )[];
 export type Truncated = [unknown, unknown] | null;
+export type SpanId7 = string | null;
 export type Timestamp7 = string;
 export type WorkingStart7 = number;
 export type Pending7 = boolean | null;
@@ -441,22 +450,26 @@ export type Decision =
   | "escalate"
   | "terminate";
 export type Explanation1 = string | null;
+export type SpanId8 = string | null;
 export type Timestamp8 = string;
 export type WorkingStart8 = number;
 export type Pending8 = boolean | null;
 export type Event8 = "input";
 export type Input4 = string;
 export type InputAnsi = string;
+export type SpanId9 = string | null;
 export type Timestamp9 = string;
 export type WorkingStart9 = number;
 export type Pending9 = boolean | null;
 export type Event9 = "score";
 export type Target2 = string | string[] | null;
 export type Intermediate = boolean;
+export type SpanId10 = string | null;
 export type Timestamp10 = string;
 export type WorkingStart10 = number;
 export type Pending10 = boolean | null;
 export type Event10 = "error";
+export type SpanId11 = string | null;
 export type Timestamp11 = string;
 export type WorkingStart11 = number;
 export type Pending11 = boolean | null;
@@ -476,24 +489,42 @@ export type Created1 = number;
 export type Filename = string;
 export type Module = string;
 export type Lineno = number;
+export type SpanId12 = string | null;
 export type Timestamp12 = string;
 export type WorkingStart12 = number;
 export type Pending12 = boolean | null;
 export type Event12 = "info";
 export type Source4 = string | null;
+export type SpanId13 = string | null;
 export type Timestamp13 = string;
 export type WorkingStart13 = number;
 export type Pending13 = boolean | null;
-export type Event13 = "step";
-export type Action1 = "begin" | "end";
+export type Event13 = "span_begin";
+export type Id8 = string;
+export type ParentId = string | null;
 export type Type13 = string | null;
 export type Name11 = string;
+export type SpanId14 = string | null;
 export type Timestamp14 = string;
 export type WorkingStart14 = number;
 export type Pending14 = boolean | null;
-export type Event14 = "subtask";
-export type Name12 = string;
+export type Event14 = "span_end";
+export type Id9 = string;
+export type SpanId15 = string | null;
+export type Timestamp15 = string;
+export type WorkingStart15 = number;
+export type Pending15 = boolean | null;
+export type Event15 = "step";
+export type Action1 = "begin" | "end";
 export type Type14 = string | null;
+export type Name12 = string;
+export type SpanId16 = string | null;
+export type Timestamp16 = string;
+export type WorkingStart16 = number;
+export type Pending16 = boolean | null;
+export type Event16 = "subtask";
+export type Name13 = string;
+export type Type15 = string | null;
 export type Events2 = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -508,6 +539,8 @@ export type Events2 = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -527,6 +560,8 @@ export type Events1 = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -548,6 +583,8 @@ export type Events = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -555,7 +592,7 @@ export type TotalTime = number | null;
 export type WorkingTime3 = number | null;
 export type Uuid = string | null;
 export type ErrorRetries = EvalError[] | null;
-export type Type15 =
+export type Type16 =
   | "context"
   | "time"
   | "working"
@@ -1121,6 +1158,7 @@ export interface Store {
  * Beginning of processing a Sample.
  */
 export interface SampleInitEvent {
+  span_id: SpanId;
   timestamp: Timestamp;
   working_start: WorkingStart;
   pending: Pending;
@@ -1145,6 +1183,7 @@ export interface Sample {
  * The sample was unable to finish processing due to a limit
  */
 export interface SampleLimitEvent {
+  span_id: SpanId1;
   timestamp: Timestamp1;
   working_start: WorkingStart1;
   pending: Pending1;
@@ -1157,6 +1196,7 @@ export interface SampleLimitEvent {
  * Sandbox execution or I/O
  */
 export interface SandboxEvent {
+  span_id: SpanId2;
   timestamp: Timestamp2;
   working_start: WorkingStart2;
   pending: Pending2;
@@ -1174,6 +1214,7 @@ export interface SandboxEvent {
  * Change to the current `TaskState`
  */
 export interface StateEvent {
+  span_id: SpanId3;
   timestamp: Timestamp3;
   working_start: WorkingStart3;
   pending: Pending3;
@@ -1198,6 +1239,7 @@ export interface JsonChange {
  * Change to data within the current `Store`.
  */
 export interface StoreEvent {
+  span_id: SpanId4;
   timestamp: Timestamp4;
   working_start: WorkingStart4;
   pending: Pending4;
@@ -1208,6 +1250,7 @@ export interface StoreEvent {
  * Call to a language model.
  */
 export interface ModelEvent {
+  span_id: SpanId5;
   timestamp: Timestamp5;
   working_start: WorkingStart5;
   pending: Pending5;
@@ -1219,6 +1262,7 @@ export interface ModelEvent {
   tool_choice: ToolChoice;
   config: GenerateConfig;
   output: ModelOutput;
+  retries: Retries;
   error: Error1;
   cache: Cache;
   call: ModelCall | null;
@@ -1289,6 +1333,7 @@ export interface Response {
  * Call to a tool.
  */
 export interface ToolEvent {
+  span_id: SpanId6;
   timestamp: Timestamp6;
   working_start: WorkingStart6;
   pending: Pending6;
@@ -1315,6 +1360,7 @@ export interface Arguments1 {
  * Tool approval.
  */
 export interface ApprovalEvent {
+  span_id: SpanId7;
   timestamp: Timestamp7;
   working_start: WorkingStart7;
   pending: Pending7;
@@ -1341,6 +1387,7 @@ export interface ToolCallView {
  * Input screen interaction.
  */
 export interface InputEvent {
+  span_id: SpanId8;
   timestamp: Timestamp8;
   working_start: WorkingStart8;
   pending: Pending8;
@@ -1355,6 +1402,7 @@ export interface InputEvent {
  * resulting from a call to `score`.
  */
 export interface ScoreEvent {
+  span_id: SpanId9;
   timestamp: Timestamp9;
   working_start: WorkingStart9;
   pending: Pending9;
@@ -1367,6 +1415,7 @@ export interface ScoreEvent {
  * Event with sample error.
  */
 export interface ErrorEvent {
+  span_id: SpanId10;
   timestamp: Timestamp10;
   working_start: WorkingStart10;
   pending: Pending10;
@@ -1377,6 +1426,7 @@ export interface ErrorEvent {
  * Log message recorded with Python logger.
  */
 export interface LoggerEvent {
+  span_id: SpanId11;
   timestamp: Timestamp11;
   working_start: WorkingStart11;
   pending: Pending11;
@@ -1399,6 +1449,7 @@ export interface LoggingMessage {
  * Event with custom info/data.
  */
 export interface InfoEvent {
+  span_id: SpanId12;
   timestamp: Timestamp12;
   working_start: WorkingStart12;
   pending: Pending12;
@@ -1407,27 +1458,54 @@ export interface InfoEvent {
   data: JsonValue;
 }
 /**
- * Step within current sample or subtask.
+ * Mark the beginning of a transcript span.
  */
-export interface StepEvent {
+export interface SpanBeginEvent {
+  span_id: SpanId13;
   timestamp: Timestamp13;
   working_start: WorkingStart13;
   pending: Pending13;
   event: Event13;
-  action: Action1;
+  id: Id8;
+  parent_id: ParentId;
   type: Type13;
   name: Name11;
 }
 /**
- * Subtask spawned.
+ * Mark the end of a transcript span.
  */
-export interface SubtaskEvent {
+export interface SpanEndEvent {
+  span_id: SpanId14;
   timestamp: Timestamp14;
   working_start: WorkingStart14;
   pending: Pending14;
   event: Event14;
-  name: Name12;
+  id: Id9;
+}
+/**
+ * Step within current sample or subtask.
+ */
+export interface StepEvent {
+  span_id: SpanId15;
+  timestamp: Timestamp15;
+  working_start: WorkingStart15;
+  pending: Pending15;
+  event: Event15;
+  action: Action1;
   type: Type14;
+  name: Name12;
+}
+/**
+ * Subtask spawned.
+ */
+export interface SubtaskEvent {
+  span_id: SpanId16;
+  timestamp: Timestamp16;
+  working_start: WorkingStart16;
+  pending: Pending16;
+  event: Event16;
+  name: Name13;
+  type: Type15;
   input: Input5;
   result: Result2;
   events: Events2;
@@ -1450,7 +1528,7 @@ export interface Attachments {
  * Limit encontered by sample.
  */
 export interface EvalSampleLimit {
-  type: Type15;
+  type: Type16;
   limit: Limit2;
 }
 /**

inspect_ai/_view/www/src/app/samples/transcript/SandboxEventView.module.css

@@ -13,7 +13,7 @@
 }
 
 .exec {
-  margin-top: 0.5em;
+  margin-top: 0;
 }
 
 .result {
@@ -29,4 +29,5 @@
   white-space: pre-wrap;
   word-wrap: break-word;
   overflow-wrap: break-word;
+  margin-bottom: 0;
 }

inspect_ai/_view/www/src/app/samples/transcript/SpanEventView.tsx

@@ -0,0 +1,174 @@
+import clsx from "clsx";
+import { FC } from "react";
+import { SpanBeginEvent } from "../../../@types/log";
+import { formatDateTime } from "../../../utils/format";
+import { EventPanel } from "./event/EventPanel";
+import { TranscriptComponent } from "./TranscriptView";
+import { kSandboxSignalName } from "./transform/fixups";
+import { EventNode } from "./types";
+
+interface SpanEventViewProps {
+  id: string;
+  event: SpanBeginEvent;
+  children: EventNode[];
+  className?: string | string[];
+}
+
+/**
+ * Renders the SpanEventView component.
+ */
+export const SpanEventView: FC<SpanEventViewProps> = ({
+  id,
+  event,
+  children,
+  className,
+}) => {
+  const descriptor = spanDescriptor(event);
+  const title =
+    descriptor.name ||
+    `${event.type ? event.type + ": " : "Step: "}${event.name}`;
+  const text = summarize(children);
+
+  return (
+    <EventPanel
+      id={`span-${event.name}-${id}`}
+      className={clsx("transcript-span", className)}
+      title={title}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      text={text}
+      collapse={descriptor.collapse}
+      icon={descriptor.icon}
+    >
+      <TranscriptComponent
+        id={`span|${event.name}|${id}`}
+        eventNodes={children}
+      />
+    </EventPanel>
+  );
+};
+
+const summarize = (children: EventNode[]) => {
+  if (children.length === 0) {
+    return "(no events)";
+  }
+
+  const formatEvent = (event: string, count: number) => {
+    if (count === 1) {
+      return `${count} ${event} event`;
+    } else {
+      return `${count} ${event} events`;
+    }
+  };
+
+  // Count the types
+  const typeCount: Record<string, number> = {};
+  children.forEach((child) => {
+    const currentCount = typeCount[child.event.event] || 0;
+    typeCount[child.event.event] = currentCount + 1;
+  });
+
+  // Try to summarize event types
+  const numberOfTypes = Object.keys(typeCount).length;
+  if (numberOfTypes < 3) {
+    return Object.keys(typeCount)
+      .map((key) => {
+        return formatEvent(key, typeCount[key]);
+      })
+      .join(", ");
+  }
+
+  // To many types, just return the number of events
+  if (children.length === 1) {
+    return "1 event";
+  } else {
+    return `${children.length} events`;
+  }
+};
+
+/**
+ * Returns a descriptor object containing icon and style based on the event type and name.
+ */
+const spanDescriptor = (
+  event: SpanBeginEvent,
+): { icon?: string; name?: string; endSpace?: boolean; collapse?: boolean } => {
+  const rootStepDescriptor = {
+    endSpace: true,
+  };
+
+  if (event.type === "solver") {
+    switch (event.name) {
+      case "chain_of_thought":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "generate":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "self_critique":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "system_message":
+        return {
+          ...rootStepDescriptor,
+          collapse: true,
+        };
+      case "use_tools":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "multiple_choice":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      default:
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+    }
+  } else if (event.type === "scorer") {
+    return {
+      ...rootStepDescriptor,
+      collapse: false,
+    };
+  } else if (event.event === "span_begin") {
+    if (event.span_id === kSandboxSignalName) {
+      return {
+        ...rootStepDescriptor,
+        name: "Sandbox Events",
+        collapse: true,
+      };
+    } else if (event.name === "init") {
+      return {
+        ...rootStepDescriptor,
+        name: "Init",
+        collapse: true,
+      };
+    } else {
+      return {
+        ...rootStepDescriptor,
+        collapse: false,
+      };
+    }
+  } else {
+    switch (event.name) {
+      case "sample_init":
+        return {
+          ...rootStepDescriptor,
+          name: "Sample Init",
+          collapse: true,
+        };
+      default:
+        return {
+          endSpace: false,
+        };
+    }
+  }
+};

inspect_ai/_view/www/src/app/samples/transcript/ToolEventView.tsx

@@ -4,7 +4,7 @@ import { resolveToolInput } from "../chat/tools/tool";
 import { ToolCallView } from "../chat/tools/ToolCallView";
 import { ApprovalEventView } from "./ApprovalEventView";
 import { EventPanel } from "./event/EventPanel";
-import { TranscriptView } from "./TranscriptView";
+import { TranscriptComponent } from "./TranscriptView";
 
 import clsx from "clsx";
 import { FC, useMemo } from "react";
@@ -12,11 +12,12 @@ import { PulsingDots } from "../../../components/PulsingDots";
 import { ChatView } from "../chat/ChatView";
 import { formatTiming, formatTitle } from "./event/utils";
 import styles from "./ToolEventView.module.css";
+import { EventNode } from "./types";
 
 interface ToolEventViewProps {
   id: string;
   event: ToolEvent;
-  depth: number;
+  children: EventNode[];
   className?: string | string[];
 }
 
@@ -26,7 +27,7 @@ interface ToolEventViewProps {
 export const ToolEventView: FC<ToolEventViewProps> = ({
   id,
   event,
-  depth,
+  children,
   className,
 }) => {
   // Extract tool input
@@ -92,13 +93,12 @@ export const ToolEventView: FC<ToolEventViewProps> = ({
           </div>
         ) : undefined}
       </div>
-      {event.events.length > 0 ? (
-        <TranscriptView
-          id={`${id}-subtask`}
+      {children.length > 0 ? (
+        <TranscriptComponent
           data-name="Transcript"
+          id={`${id}-subtask`}
+          eventNodes={children}
           data-default={event.failed || event.agent ? true : null}
-          events={event.events}
-          depth={depth + 1}
         />
       ) : (
         ""

inspect_ai/_view/www/src/app/samples/transcript/TranscriptView.tsx

@@ -17,6 +17,7 @@ import { ToolEventView } from "./ToolEventView";
 import { EventNode } from "./types";
 
 import clsx from "clsx";
+import { SpanEventView } from "./SpanEventView";
 import styles from "./TranscriptView.module.css";
 import { TranscriptVirtualListComponent } from "./TranscriptVirtualListComponent";
 import { fixupEventStream } from "./transform/fixups";
@@ -64,7 +65,6 @@ export const TranscriptVirtualList: FC<TranscriptVirtualListProps> = memo(
     const eventNodes = useMemo(() => {
       const resolvedEvents = fixupEventStream(events, !running);
       const eventNodes = treeifyEvents(resolvedEvents, depth || 0);
-
       return eventNodes;
     }, [events, depth]);
 
@@ -201,6 +201,16 @@ export const RenderedEventNode: FC<RenderedEventNodeProps> = memo(
           <StateEventView id={id} event={node.event} className={className} />
         );
 
+      case "span_begin":
+        return (
+          <SpanEventView
+            id={id}
+            event={node.event}
+            children={node.children}
+            className={className}
+          />
+        );
+
       case "step":
         return (
           <StepEventView
@@ -237,7 +247,7 @@ export const RenderedEventNode: FC<RenderedEventNodeProps> = memo(
             id={id}
             event={node.event}
             className={className}
-            depth={node.depth}
+            children={node.children}
           />
         );
 

inspect_ai/_view/www/src/app/samples/transcript/TranscriptVirtualListComponent.module.css

@@ -8,7 +8,7 @@
 
 .node {
   padding-top: 0.7rem;
-  padding-bottom: 0em;
+  padding-bottom: 1px;
 }
 
 .attached {

inspect_ai/_view/www/src/app/samples/transcript/event/EventPanel.tsx

@@ -9,7 +9,6 @@ import {
 import { ApplicationIcons } from "../../../appearance/icons";
 import { EventNavs } from "./EventNavs";
 
-import { ProgressBar } from "../../../../components/ProgressBar";
 import { useProperty } from "../../../../state/hooks";
 import styles from "./EventPanel.module.css";
 
@@ -41,7 +40,6 @@ export const EventPanel: FC<EventPanelProps> = ({
   icon,
   collapse,
   children,
-  running,
 }) => {
   const [isCollapsed, setCollapsed] = useProperty(id, "collapsed", {
     defaultValue: !!collapse,
@@ -191,7 +189,6 @@ export const EventPanel: FC<EventPanelProps> = ({
           })}
         </div>
       </div>
-      <ProgressBar animating={!!running} />
     </>
   );
   return card;