inspect-ai 0.3.92__py3-none-any.whl → 0.3.94__py3-none-any.whl

inspect_ai/_view/www/src/app/samples/transcript/transform/fixups.ts

@@ -1,6 +1,12 @@
 // This is a special name that signals a group of sandbox events.
 
-import { Events, StepEvent } from "../../../../@types/log";
+import {
+  Events,
+  SpanBeginEvent,
+  SpanEndEvent,
+  StepEvent,
+} from "../../../../@types/log";
+import { hasSpans } from "./utils";
 
 // It will be caught elsewhere and rendered with a pretty name
 export const kSandboxSignalName = "53787D8A-D3FC-426D-B383-9F880B70E4AA";
@@ -54,39 +60,55 @@ const processPendingEvents = (events: Events, filter: boolean): Events => {
 };
 
 const collapseSampleInit = (events: Events): Events => {
-  // See if the events have an init step
+  // Don't performance sample init logic if spans are present
+  const hasSpans = events.some((e) => {
+    return e.event === "span_begin" || e.event === "span_end";
+  });
+  if (hasSpans) {
+    return events;
+  }
+
+  // Don't synthesize a sample init step if one already exists
   const hasInitStep =
     events.findIndex((e) => {
       return e.event === "step" && e.name === "init";
     }) !== -1;
+  if (hasInitStep) {
+    return events;
+  }
 
+  // Find a sample init event
   const initEventIndex = events.findIndex((e) => {
     return e.event === "sample_init";
   });
   const initEvent = events[initEventIndex];
+  if (!initEvent) {
+    return events;
+  }
 
+  // Splice in sample init step if needed
   const fixedUp = [...events];
-  if (!hasInitStep && initEvent) {
-    fixedUp.splice(initEventIndex, 0, {
-      timestamp: initEvent.timestamp,
-      event: "step",
-      action: "begin",
-      type: null,
-      name: "sample_init",
-      pending: false,
-      working_start: 0,
-    });
-
-    fixedUp.splice(initEventIndex + 2, 0, {
-      timestamp: initEvent.timestamp,
-      event: "step",
-      action: "end",
-      type: null,
-      name: "sample_init",
-      pending: false,
-      working_start: 0,
-    });
-  }
+  fixedUp.splice(initEventIndex, 0, {
+    timestamp: initEvent.timestamp,
+    event: "step",
+    action: "begin",
+    type: null,
+    name: "sample_init",
+    pending: false,
+    working_start: 0,
+    span_id: initEvent.span_id,
+  });
+
+  fixedUp.splice(initEventIndex + 2, 0, {
+    timestamp: initEvent.timestamp,
+    event: "step",
+    action: "end",
+    type: null,
+    name: "sample_init",
+    pending: false,
+    working_start: 0,
+    span_id: initEvent.span_id,
+  });
   return fixedUp;
 };
 
@@ -94,12 +116,22 @@ const groupSandboxEvents = (events: Events): Events => {
   const result: Events = [];
   const pendingSandboxEvents: Events = [];
 
+  const useSpans = hasSpans(events);
+
   const pushPendingSandboxEvents = () => {
     const timestamp =
       pendingSandboxEvents[pendingSandboxEvents.length - 1].timestamp;
-    result.push(createStepEvent(kSandboxSignalName, timestamp, "begin"));
+    if (useSpans) {
+      result.push(createSpanBegin(kSandboxSignalName, timestamp, null));
+    } else {
+      result.push(createStepEvent(kSandboxSignalName, timestamp, "begin"));
+    }
     result.push(...pendingSandboxEvents);
-    result.push(createStepEvent(kSandboxSignalName, timestamp, "end"));
+    if (useSpans) {
+      result.push(createSpanEnd(kSandboxSignalName, timestamp));
+    } else {
+      result.push(createStepEvent(kSandboxSignalName, timestamp, "end"));
+    }
     pendingSandboxEvents.length = 0;
   };
 
@@ -139,4 +171,34 @@ const createStepEvent = (
   name,
   pending: false,
   working_start: 0,
+  span_id: null,
 });
+
+const createSpanBegin = (
+  name: string,
+  timestamp: string,
+  parent_id: string | null,
+): SpanBeginEvent => {
+  return {
+    name,
+    id: `${name}-begin`,
+    span_id: name,
+    parent_id,
+    timestamp,
+    event: "span_begin",
+    type: null,
+    pending: false,
+    working_start: 0,
+  };
+};
+
+const createSpanEnd = (name: string, timestamp: string): SpanEndEvent => {
+  return {
+    id: `${name}-end`,
+    timestamp,
+    event: "span_end",
+    pending: false,
+    working_start: 0,
+    span_id: name,
+  };
+};

inspect_ai/_view/www/src/app/samples/transcript/transform/treeify.ts

@@ -1,14 +1,28 @@
 import { Events } from "../../../../@types/log";
 import { EventNode, EventType } from "../types";
+import {
+  ACTION_BEGIN,
+  ET_SPAN_BEGIN,
+  ET_SPAN_END,
+  ET_STEP,
+  hasSpans,
+} from "./utils";
+
+type TreeifyFunction = (
+  event: EventType,
+  addNode: (event: EventType) => EventNode,
+  pushStack: (node: EventNode) => void,
+  popStack: () => void,
+) => void;
 
-/**
- * Gathers events into a hierarchy of EventNodes.
- */
 export function treeifyEvents(events: Events, depth: number): EventNode[] {
+  const useSpans = hasSpans(events);
+  const treeFn = useSpans ? treeifyFnSpan : treeifyFnStep;
+
   const rootNodes: EventNode[] = [];
   const stack: EventNode[] = [];
 
-  const pushNode = (event: EventType): EventNode => {
+  const addNode = (event: EventType): EventNode => {
     const node = new EventNode(event, stack.length + depth);
     if (stack.length > 0) {
       const parentNode = stack[stack.length - 1];
@@ -19,21 +33,219 @@ export function treeifyEvents(events: Events, depth: number): EventNode[] {
     return node;
   };
 
+  const pushStack = (node: EventNode): void => {
+    stack.push(node);
+  };
+
+  const popStack = (): void => {
+    if (stack.length > 0) {
+      stack.pop();
+    }
+  };
+
   events.forEach((event) => {
-    if (event.event === "step" && event.action === "begin") {
-      // Starting a new step
-      const node = pushNode(event);
-      stack.push(node);
-    } else if (event.event === "step" && event.action === "end") {
-      // An ending step
-      if (stack.length > 0) {
-        stack.pop();
+    treeFn(event, addNode, pushStack, popStack);
+  });
+
+  if (useSpans) {
+    return transformTree(rootNodes);
+  } else {
+    return rootNodes;
+  }
+}
+
+const treeifyFnStep: TreeifyFunction = (
+  event: EventType,
+  addNode: (event: EventType) => EventNode,
+  pushStack: (node: EventNode) => void,
+  popStack: () => void,
+): void => {
+  switch (event.event) {
+    case ET_STEP:
+      if (event.action === ACTION_BEGIN) {
+        // Starting a new step
+        const node = addNode(event);
+        pushStack(node);
+      } else {
+        // An ending step
+        popStack();
       }
-    } else {
+      break;
+    case ET_SPAN_BEGIN: {
+      // These shoudn't be here, but throw away
+      break;
+    }
+    case ET_SPAN_END: {
+      // These shoudn't be here, but throw away
+      break;
+    }
+    default:
       // An event
-      pushNode(event);
+      addNode(event);
+      break;
+  }
+};
+
+const treeifyFnSpan: TreeifyFunction = (
+  event: EventType,
+  addNode: (event: EventType) => EventNode,
+  pushStack: (node: EventNode) => void,
+  popStack: () => void,
+): void => {
+  switch (event.event) {
+    case ET_STEP:
+      // strip steps
+      break;
+    case ET_SPAN_BEGIN: {
+      const node = addNode(event);
+      pushStack(node);
+      break;
     }
-  });
+    case ET_SPAN_END: {
+      popStack();
+      break;
+    }
+    default:
+      // An event
+      addNode(event);
+      break;
+  }
+};
 
-  return rootNodes;
-}
+type TreeNodeTransformer = {
+  name: string;
+  matches: (node: EventNode) => boolean;
+  process: (node: EventNode) => EventNode;
+};
+
+const treeNodeTransformers: TreeNodeTransformer[] = [
+  {
+    name: "unwrap_tools",
+    matches: (node) =>
+      node.event.event === "span_begin" && node.event.type === "tool",
+    process: (node) => elevateChildNode(node, "tool") || node,
+  },
+  {
+    name: "unwrap_subtasks",
+    matches: (node) =>
+      node.event.event === "span_begin" && node.event.type === "subtask",
+    process: (node) => elevateChildNode(node, "subtask") || node,
+  },
+  {
+    name: "unwrap_agent_solver",
+    matches: (node) =>
+      node.event.event === "span_begin" &&
+      node.event["type"] === "solver" &&
+      node.children.length === 2 &&
+      node.children[0].event.event === "span_begin" &&
+      node.children[0].event.type === "agent" &&
+      node.children[1].event.event === "state",
+
+    process: (node) => skipFirstChildNode(node),
+  },
+  {
+    name: "unwrap_agent_solver w/store",
+    matches: (node) =>
+      node.event.event === "span_begin" &&
+      node.event["type"] === "solver" &&
+      node.children.length === 3 &&
+      node.children[0].event.event === "span_begin" &&
+      node.children[0].event.type === "agent" &&
+      node.children[1].event.event === "state" &&
+      node.children[2].event.event === "store",
+    process: (node) => skipFirstChildNode(node),
+  },
+  {
+    name: "unwrap_handoff",
+    matches: (node) =>
+      node.event.event === "span_begin" &&
+      node.event["type"] === "handoff" &&
+      node.children.length === 2 &&
+      node.children[0].event.event === "tool" &&
+      node.children[1].event.event === "store" &&
+      node.children[0].children.length === 2 &&
+      node.children[0].children[0].event.event === "span_begin" &&
+      node.children[0].children[0].event.type === "agent",
+    process: (node) => skipThisNode(node),
+  },
+];
+
+const transformTree = (roots: EventNode[]): EventNode[] => {
+  const visitNode = (node: EventNode): EventNode => {
+    let processedNode = node;
+
+    // Visit children (depth first)
+    processedNode.children = processedNode.children.map(visitNode);
+
+    // Apply any visitors to this node
+    for (const transformer of treeNodeTransformers) {
+      if (transformer.matches(processedNode)) {
+        processedNode = transformer.process(processedNode);
+        // Only apply the first matching transformer
+        break;
+      }
+    }
+    return processedNode;
+  };
+
+  return roots.map(visitNode);
+};
+
+/**
+ * Process a span node by elevating a specific child node type and moving its siblings as children
+ * @template T - Type of the event (either ToolEvent or SubtaskEvent)
+ */
+const elevateChildNode = (
+  node: EventNode,
+  childEventType: "tool" | "subtask",
+): EventNode | null => {
+  // Find the specific event child
+  const targetIndex = node.children.findIndex(
+    (child) => child.event.event === childEventType,
+  );
+
+  if (targetIndex === -1) {
+    console.log(
+      `No ${childEventType} event found in a span, this is very unexpected.`,
+    );
+    return null;
+  }
+
+  // Get the target node and set its depth
+  const targetNode = { ...node.children[targetIndex] };
+  const remainingChildren = node.children.filter((_, i) => i !== targetIndex);
+
+  // Process the remaining children
+  targetNode.depth = node.depth;
+  targetNode.children = reduceDepth(remainingChildren);
+
+  // No need to update the event itself (events have been deprecated
+  // and more importantly we drive children / transcripts using the tree structure itself
+  // and notes rather than the event.events itself)
+  return targetNode;
+};
+
+const skipFirstChildNode = (node: EventNode): EventNode => {
+  const agentSpan = node.children.splice(0, 1)[0];
+  node.children.unshift(...reduceDepth(agentSpan.children));
+  return node;
+};
+
+const skipThisNode = (node: EventNode): EventNode => {
+  const newNode = { ...node.children[0] };
+  newNode.depth = node.depth;
+  newNode.children = reduceDepth(newNode.children[0].children, 2);
+  return newNode;
+};
+
+// Reduce the depth of the children by 1
+// This is used when we hoist a child node to the parent
+const reduceDepth = (nodes: EventNode[], depth: number = 1): EventNode[] => {
+  return nodes.map((node) => {
+    if (node.children.length > 0) {
+      node.children = reduceDepth(node.children, 1);
+    }
+    node.depth = node.depth - depth;
+    return node;
+  });
+};

inspect_ai/_view/www/src/app/samples/transcript/transform/utils.ts

@@ -0,0 +1,11 @@
+import { Events } from "../../../../@types/log";
+
+export const ET_STEP = "step";
+export const ACTION_BEGIN = "begin";
+
+export const ET_SPAN_BEGIN = "span_begin";
+export const ET_SPAN_END = "span_end";
+
+export const hasSpans = (events: Events): boolean => {
+  return events.some((event) => event.event === ET_SPAN_BEGIN);
+};

inspect_ai/_view/www/src/app/samples/transcript/types.ts

@@ -10,6 +10,8 @@ import {
   SampleLimitEvent,
   SandboxEvent,
   ScoreEvent,
+  SpanBeginEvent,
+  SpanEndEvent,
   StateEvent,
   StepEvent,
   StoreEvent,
@@ -39,7 +41,9 @@ export type EventType =
   | InputEvent
   | ErrorEvent
   | ApprovalEvent
-  | SandboxEvent;
+  | SandboxEvent
+  | SpanBeginEvent
+  | SpanEndEvent;
 
 export class EventNode {
   event: EventType;

inspect_ai/_view/www/src/app/usage/ModelUsagePanel.tsx

@@ -6,6 +6,7 @@ import styles from "./ModelUsagePanel.module.css";
 
 interface ModelUsageProps {
   usage: ModelUsage1;
+  className?: string | string[];
 }
 
 interface ModelUsageRow {
@@ -19,7 +20,7 @@ interface ModelUsageRow {
 /**
  * Renders the ModelUsagePanel component.
  */
-export const ModelUsagePanel: FC<ModelUsageProps> = ({ usage }) => {
+export const ModelUsagePanel: FC<ModelUsageProps> = ({ usage, className }) => {
   if (!usage) {
     return null;
   }
@@ -84,7 +85,7 @@ export const ModelUsagePanel: FC<ModelUsageProps> = ({ usage }) => {
   });
 
   return (
-    <div className={clsx("text-size-small", styles.wrapper)}>
+    <div className={clsx("text-size-small", styles.wrapper, className)}>
       {rows.map((row, idx) => {
         if (row.label === "---") {
           return (

inspect_ai/_view/www/src/app/usage/TokenTable.module.css

@@ -1,6 +1,5 @@
 .table {
   width: 100%;
-  margin-top: 0.7rem;
 }
 
 .tableTokens {
@@ -15,3 +14,7 @@
 .model {
   padding-right: 1em;
 }
+
+.cellContents {
+  padding-bottom: 1em;
+}

inspect_ai/_view/www/src/app/usage/TokenTable.tsx

@@ -79,10 +79,10 @@ export const TokenRow: FC<TokenRowProps> = ({ model, usage }) => {
   return (
     <tr>
       <td>
-        <div className={styles.model}>{model}</div>
+        <div className={clsx(styles.model, styles.cellContents)}>{model}</div>
       </td>
       <td>
-        <ModelUsagePanel usage={usage} />
+        <ModelUsagePanel usage={usage} className={clsx(styles.cellContents)} />
       </td>
     </tr>
   );

inspect_ai/_view/www/src/app/usage/UsageCard.module.css

@@ -3,13 +3,18 @@
   padding-bottom: 1em;
   margin-left: 0.5em;
   display: flex;
+  flex-wrap: wrap;
+  gap: 1em;
 }
 
 .col1 {
-  flex: 1 1 40%;
-  margin-right: 1em;
+  flex: 0 1 auto;
+  min-width: 200px;
+  width: fit-content;
 }
 
 .col2 {
-  flex: 1 1 60%;
+  flex: 1 1 auto;
+  min-width: 300px;
+  width: fit-content;
 }

inspect_ai/_view/www/src/app/usage/UsageCard.tsx

@@ -1,10 +1,5 @@
-import clsx from "clsx";
 import { EvalStats } from "../../@types/log";
-import { FontSize } from "../../app/appearance/fonts";
-import { ApplicationIcons } from "../../app/appearance/icons";
-import { MetaDataView } from "../../app/content/MetaDataView";
 import { Card, CardBody, CardHeader } from "../../components/Card";
-import { formatDuration } from "../../utils/format";
 import { ModelTokenTable } from "./ModelTokenTable";
 
 import { FC } from "react";
@@ -24,40 +19,11 @@ export const UsageCard: FC<UsageCardProps> = ({ stats }) => {
     return null;
   }
 
-  const totalDuration = formatDuration(
-    new Date(stats.started_at),
-    new Date(stats.completed_at),
-  );
-  const usageMetadataStyle = {
-    fontSize: FontSize.smaller,
-  };
-
   return (
     <Card>
-      <CardHeader icon={ApplicationIcons.usage} label="Usage" />
+      <CardHeader label="Usage" />
       <CardBody id={kUsageCardBodyId}>
         <div className={styles.wrapper}>
-          <div className={styles.col1}>
-            <div
-              className={clsx(
-                "text-size-smaller",
-                "text-style-label",
-                "text-style-secondary",
-              )}
-            >
-              Duration
-            </div>
-            <MetaDataView
-              entries={{
-                ["Start"]: new Date(stats.started_at).toLocaleString(),
-                ["End"]: new Date(stats.completed_at).toLocaleString(),
-                ["Duration"]: totalDuration,
-              }}
-              tableOptions="borderless,sm"
-              style={usageMetadataStyle}
-            />
-          </div>
-
           <div className={styles.col2}>
             <ModelTokenTable model_usage={stats.model_usage} />
           </div>

inspect_ai/_view/www/src/components/Card.css

@@ -23,7 +23,6 @@
   background-color: var(--bs-light-bg-subtle);
   border: solid 1px var(--bs-light-border-subtle);
   border-radius: var(--bs-border-radius);
-  margin-bottom: 1.5em;
 }
 
 .card-collaping-header {

inspect_ai/_view/www/src/constants.ts

@@ -5,6 +5,8 @@ export const kModelNone = "none/none";
 export const kLogViewSamplesTabId = "samples";
 export const kLogViewJsonTabId = "json";
 export const kLogViewInfoTabId = "info";
+export const kLogViewModelsTabId = "models";
+export const kLogViewTaskTabId = "task";
 
 // Sample tab constants
 export const kSampleMessagesTabId = `messages`;

inspect_ai/_view/www/src/utils/numeric.ts

@@ -0,0 +1,17 @@
+export function compareWithNan(a: number, b: number): number {
+  const aIsNaN = Number.isNaN(a);
+  const bIsNaN = Number.isNaN(b);
+
+  if (aIsNaN && bIsNaN) {
+    return 0;
+  }
+
+  if (aIsNaN) {
+    return 1;
+  }
+  if (bIsNaN) {
+    return -1;
+  }
+
+  return a - b;
+}

inspect_ai/agent/_agent.py

@@ -27,13 +27,14 @@ from inspect_ai.model._chat_message import (
     ChatMessageAssistant,
 )
 from inspect_ai.model._model_output import ChatCompletionChoice, ModelOutput
+from inspect_ai.util._limited_conversation import ChatMessageList
 
 
 class AgentState:
     """Agent state."""
 
     def __init__(self, *, messages: list[ChatMessage]) -> None:
-        self._messages = messages
+        self._messages: list[ChatMessage] = ChatMessageList(messages)
         self._output: ModelOutput | None = None
 
     @property
@@ -43,8 +44,7 @@ class AgentState:
 
     @messages.setter
     def messages(self, messages: list[ChatMessage]) -> None:
-        """Set the conversation history."""
-        self._messages = messages
+        self._messages = ChatMessageList(messages)
 
     @property
     def output(self) -> ModelOutput: