inspect-ai 0.3.93__py3-none-any.whl → 0.3.95__py3-none-any.whl

inspect_ai/_view/www/src/app/samples/transcript/transform/fixups.ts

@@ -1,6 +1,12 @@
 // This is a special name that signals a group of sandbox events.
 
-import { Events, StepEvent } from "../../../../@types/log";
+import {
+  Events,
+  SpanBeginEvent,
+  SpanEndEvent,
+  StepEvent,
+} from "../../../../@types/log";
+import { hasSpans } from "./utils";
 
 // It will be caught elsewhere and rendered with a pretty name
 export const kSandboxSignalName = "53787D8A-D3FC-426D-B383-9F880B70E4AA";
@@ -54,39 +60,55 @@ const processPendingEvents = (events: Events, filter: boolean): Events => {
 };
 
 const collapseSampleInit = (events: Events): Events => {
-  // See if the events have an init step
+  // Don't performance sample init logic if spans are present
+  const hasSpans = events.some((e) => {
+    return e.event === "span_begin" || e.event === "span_end";
+  });
+  if (hasSpans) {
+    return events;
+  }
+
+  // Don't synthesize a sample init step if one already exists
   const hasInitStep =
     events.findIndex((e) => {
       return e.event === "step" && e.name === "init";
     }) !== -1;
+  if (hasInitStep) {
+    return events;
+  }
 
+  // Find a sample init event
   const initEventIndex = events.findIndex((e) => {
     return e.event === "sample_init";
   });
   const initEvent = events[initEventIndex];
+  if (!initEvent) {
+    return events;
+  }
 
+  // Splice in sample init step if needed
   const fixedUp = [...events];
-  if (!hasInitStep && initEvent) {
-    fixedUp.splice(initEventIndex, 0, {
-      timestamp: initEvent.timestamp,
-      event: "step",
-      action: "begin",
-      type: null,
-      name: "sample_init",
-      pending: false,
-      working_start: 0,
-    });
-
-    fixedUp.splice(initEventIndex + 2, 0, {
-      timestamp: initEvent.timestamp,
-      event: "step",
-      action: "end",
-      type: null,
-      name: "sample_init",
-      pending: false,
-      working_start: 0,
-    });
-  }
+  fixedUp.splice(initEventIndex, 0, {
+    timestamp: initEvent.timestamp,
+    event: "step",
+    action: "begin",
+    type: null,
+    name: "sample_init",
+    pending: false,
+    working_start: 0,
+    span_id: initEvent.span_id,
+  });
+
+  fixedUp.splice(initEventIndex + 2, 0, {
+    timestamp: initEvent.timestamp,
+    event: "step",
+    action: "end",
+    type: null,
+    name: "sample_init",
+    pending: false,
+    working_start: 0,
+    span_id: initEvent.span_id,
+  });
   return fixedUp;
 };
 
@@ -94,12 +116,22 @@ const groupSandboxEvents = (events: Events): Events => {
   const result: Events = [];
   const pendingSandboxEvents: Events = [];
 
+  const useSpans = hasSpans(events);
+
   const pushPendingSandboxEvents = () => {
     const timestamp =
       pendingSandboxEvents[pendingSandboxEvents.length - 1].timestamp;
-    result.push(createStepEvent(kSandboxSignalName, timestamp, "begin"));
+    if (useSpans) {
+      result.push(createSpanBegin(kSandboxSignalName, timestamp, null));
+    } else {
+      result.push(createStepEvent(kSandboxSignalName, timestamp, "begin"));
+    }
     result.push(...pendingSandboxEvents);
-    result.push(createStepEvent(kSandboxSignalName, timestamp, "end"));
+    if (useSpans) {
+      result.push(createSpanEnd(kSandboxSignalName, timestamp));
+    } else {
+      result.push(createStepEvent(kSandboxSignalName, timestamp, "end"));
+    }
     pendingSandboxEvents.length = 0;
   };
 
@@ -139,4 +171,34 @@ const createStepEvent = (
   name,
   pending: false,
   working_start: 0,
+  span_id: null,
 });
+
+const createSpanBegin = (
+  name: string,
+  timestamp: string,
+  parent_id: string | null,
+): SpanBeginEvent => {
+  return {
+    name,
+    id: `${name}-begin`,
+    span_id: name,
+    parent_id,
+    timestamp,
+    event: "span_begin",
+    type: null,
+    pending: false,
+    working_start: 0,
+  };
+};
+
+const createSpanEnd = (name: string, timestamp: string): SpanEndEvent => {
+  return {
+    id: `${name}-end`,
+    timestamp,
+    event: "span_end",
+    pending: false,
+    working_start: 0,
+    span_id: name,
+  };
+};

inspect_ai/_view/www/src/app/samples/transcript/transform/treeify.ts

@@ -1,14 +1,28 @@
 import { Events } from "../../../../@types/log";
 import { EventNode, EventType } from "../types";
+import {
+  ACTION_BEGIN,
+  ET_SPAN_BEGIN,
+  ET_SPAN_END,
+  ET_STEP,
+  hasSpans,
+} from "./utils";
+
+type TreeifyFunction = (
+  event: EventType,
+  addNode: (event: EventType) => EventNode,
+  pushStack: (node: EventNode) => void,
+  popStack: () => void,
+) => void;
 
-/**
- * Gathers events into a hierarchy of EventNodes.
- */
 export function treeifyEvents(events: Events, depth: number): EventNode[] {
+  const useSpans = hasSpans(events);
+  const treeFn = useSpans ? treeifyFnSpan : treeifyFnStep;
+
   const rootNodes: EventNode[] = [];
   const stack: EventNode[] = [];
 
-  const pushNode = (event: EventType): EventNode => {
+  const addNode = (event: EventType): EventNode => {
     const node = new EventNode(event, stack.length + depth);
     if (stack.length > 0) {
       const parentNode = stack[stack.length - 1];
@@ -19,21 +33,219 @@ export function treeifyEvents(events: Events, depth: number): EventNode[] {
     return node;
   };
 
+  const pushStack = (node: EventNode): void => {
+    stack.push(node);
+  };
+
+  const popStack = (): void => {
+    if (stack.length > 0) {
+      stack.pop();
+    }
+  };
+
   events.forEach((event) => {
-    if (event.event === "step" && event.action === "begin") {
-      // Starting a new step
-      const node = pushNode(event);
-      stack.push(node);
-    } else if (event.event === "step" && event.action === "end") {
-      // An ending step
-      if (stack.length > 0) {
-        stack.pop();
+    treeFn(event, addNode, pushStack, popStack);
+  });
+
+  if (useSpans) {
+    return transformTree(rootNodes);
+  } else {
+    return rootNodes;
+  }
+}
+
+const treeifyFnStep: TreeifyFunction = (
+  event: EventType,
+  addNode: (event: EventType) => EventNode,
+  pushStack: (node: EventNode) => void,
+  popStack: () => void,
+): void => {
+  switch (event.event) {
+    case ET_STEP:
+      if (event.action === ACTION_BEGIN) {
+        // Starting a new step
+        const node = addNode(event);
+        pushStack(node);
+      } else {
+        // An ending step
+        popStack();
       }
-    } else {
+      break;
+    case ET_SPAN_BEGIN: {
+      // These shoudn't be here, but throw away
+      break;
+    }
+    case ET_SPAN_END: {
+      // These shoudn't be here, but throw away
+      break;
+    }
+    default:
       // An event
-      pushNode(event);
+      addNode(event);
+      break;
+  }
+};
+
+const treeifyFnSpan: TreeifyFunction = (
+  event: EventType,
+  addNode: (event: EventType) => EventNode,
+  pushStack: (node: EventNode) => void,
+  popStack: () => void,
+): void => {
+  switch (event.event) {
+    case ET_STEP:
+      // strip steps
+      break;
+    case ET_SPAN_BEGIN: {
+      const node = addNode(event);
+      pushStack(node);
+      break;
     }
-  });
+    case ET_SPAN_END: {
+      popStack();
+      break;
+    }
+    default:
+      // An event
+      addNode(event);
+      break;
+  }
+};
 
-  return rootNodes;
-}
+type TreeNodeTransformer = {
+  name: string;
+  matches: (node: EventNode) => boolean;
+  process: (node: EventNode) => EventNode;
+};
+
+const treeNodeTransformers: TreeNodeTransformer[] = [
+  {
+    name: "unwrap_tools",
+    matches: (node) =>
+      node.event.event === "span_begin" && node.event.type === "tool",
+    process: (node) => elevateChildNode(node, "tool") || node,
+  },
+  {
+    name: "unwrap_subtasks",
+    matches: (node) =>
+      node.event.event === "span_begin" && node.event.type === "subtask",
+    process: (node) => elevateChildNode(node, "subtask") || node,
+  },
+  {
+    name: "unwrap_agent_solver",
+    matches: (node) =>
+      node.event.event === "span_begin" &&
+      node.event["type"] === "solver" &&
+      node.children.length === 2 &&
+      node.children[0].event.event === "span_begin" &&
+      node.children[0].event.type === "agent" &&
+      node.children[1].event.event === "state",
+
+    process: (node) => skipFirstChildNode(node),
+  },
+  {
+    name: "unwrap_agent_solver w/store",
+    matches: (node) =>
+      node.event.event === "span_begin" &&
+      node.event["type"] === "solver" &&
+      node.children.length === 3 &&
+      node.children[0].event.event === "span_begin" &&
+      node.children[0].event.type === "agent" &&
+      node.children[1].event.event === "state" &&
+      node.children[2].event.event === "store",
+    process: (node) => skipFirstChildNode(node),
+  },
+  {
+    name: "unwrap_handoff",
+    matches: (node) =>
+      node.event.event === "span_begin" &&
+      node.event["type"] === "handoff" &&
+      node.children.length === 2 &&
+      node.children[0].event.event === "tool" &&
+      node.children[1].event.event === "store" &&
+      node.children[0].children.length === 2 &&
+      node.children[0].children[0].event.event === "span_begin" &&
+      node.children[0].children[0].event.type === "agent",
+    process: (node) => skipThisNode(node),
+  },
+];
+
+const transformTree = (roots: EventNode[]): EventNode[] => {
+  const visitNode = (node: EventNode): EventNode => {
+    let processedNode = node;
+
+    // Visit children (depth first)
+    processedNode.children = processedNode.children.map(visitNode);
+
+    // Apply any visitors to this node
+    for (const transformer of treeNodeTransformers) {
+      if (transformer.matches(processedNode)) {
+        processedNode = transformer.process(processedNode);
+        // Only apply the first matching transformer
+        break;
+      }
+    }
+    return processedNode;
+  };
+
+  return roots.map(visitNode);
+};
+
+/**
+ * Process a span node by elevating a specific child node type and moving its siblings as children
+ * @template T - Type of the event (either ToolEvent or SubtaskEvent)
+ */
+const elevateChildNode = (
+  node: EventNode,
+  childEventType: "tool" | "subtask",
+): EventNode | null => {
+  // Find the specific event child
+  const targetIndex = node.children.findIndex(
+    (child) => child.event.event === childEventType,
+  );
+
+  if (targetIndex === -1) {
+    console.log(
+      `No ${childEventType} event found in a span, this is very unexpected.`,
+    );
+    return null;
+  }
+
+  // Get the target node and set its depth
+  const targetNode = { ...node.children[targetIndex] };
+  const remainingChildren = node.children.filter((_, i) => i !== targetIndex);
+
+  // Process the remaining children
+  targetNode.depth = node.depth;
+  targetNode.children = reduceDepth(remainingChildren);
+
+  // No need to update the event itself (events have been deprecated
+  // and more importantly we drive children / transcripts using the tree structure itself
+  // and notes rather than the event.events itself)
+  return targetNode;
+};
+
+const skipFirstChildNode = (node: EventNode): EventNode => {
+  const agentSpan = node.children.splice(0, 1)[0];
+  node.children.unshift(...reduceDepth(agentSpan.children));
+  return node;
+};
+
+const skipThisNode = (node: EventNode): EventNode => {
+  const newNode = { ...node.children[0] };
+  newNode.depth = node.depth;
+  newNode.children = reduceDepth(newNode.children[0].children, 2);
+  return newNode;
+};
+
+// Reduce the depth of the children by 1
+// This is used when we hoist a child node to the parent
+const reduceDepth = (nodes: EventNode[], depth: number = 1): EventNode[] => {
+  return nodes.map((node) => {
+    if (node.children.length > 0) {
+      node.children = reduceDepth(node.children, 1);
+    }
+    node.depth = node.depth - depth;
+    return node;
+  });
+};

inspect_ai/_view/www/src/app/samples/transcript/transform/utils.ts

@@ -0,0 +1,11 @@
+import { Events } from "../../../../@types/log";
+
+export const ET_STEP = "step";
+export const ACTION_BEGIN = "begin";
+
+export const ET_SPAN_BEGIN = "span_begin";
+export const ET_SPAN_END = "span_end";
+
+export const hasSpans = (events: Events): boolean => {
+  return events.some((event) => event.event === ET_SPAN_BEGIN);
+};

inspect_ai/_view/www/src/app/samples/transcript/types.ts

@@ -10,6 +10,8 @@ import {
   SampleLimitEvent,
   SandboxEvent,
   ScoreEvent,
+  SpanBeginEvent,
+  SpanEndEvent,
   StateEvent,
   StepEvent,
   StoreEvent,
@@ -39,7 +41,9 @@ export type EventType =
   | InputEvent
   | ErrorEvent
   | ApprovalEvent
-  | SandboxEvent;
+  | SandboxEvent
+  | SpanBeginEvent
+  | SpanEndEvent;
 
 export class EventNode {
   event: EventType;

inspect_ai/_view/www/src/app/types.ts

@@ -67,7 +67,9 @@ export interface LogState {
   selectedLogSummary?: EvalSummary;
   pendingSampleSummaries?: PendingSamples;
 
-  filter: ScoreFilter;
+  filter: string;
+  filterError?: FilterError;
+
   epoch: string;
   sort: string;
   score?: ScoreLabel;
@@ -122,8 +124,16 @@ export interface ScoreLabel {
   scorer: string;
 }
 
-export interface ScoreFilter {
+export interface SampleFilter {
   value?: string;
+  error?: FilterError;
+}
+
+export interface FilterError {
+  from: number;
+  to: number;
+  message: string;
+  severity: "warning" | "error";
 }
 
 export type SampleMode = "none" | "single" | "many";

inspect_ai/_view/www/src/components/ExpandablePanel.module.css

@@ -18,7 +18,7 @@
   display: flex;
   margin-top: 0;
   position: relative;
-  height: 8px;
+  height: 18px;
 }
 
 .moreToggle.bordered {

inspect_ai/_view/www/src/components/ExpandablePanel.tsx

@@ -27,19 +27,19 @@ export const ExpandablePanel: FC<ExpandablePanelProps> = memo(
     const [collapsed, setCollapsed] = useCollapsedState(id, collapse);
 
     const [showToggle, setShowToggle] = useState(false);
-    const lineHeightRef = useRef<number>(0);
+    const baseFontSizeRef = useRef<number>(0);
 
     const checkOverflow = useCallback(
       (entry: ResizeObserverEntry) => {
         const element = entry.target as HTMLDivElement;
 
         // Calculate line height if we haven't yet
-        if (!lineHeightRef.current) {
+        if (baseFontSizeRef.current === 0) {
           const computedStyle = window.getComputedStyle(element);
-          lineHeightRef.current = parseInt(computedStyle.lineHeight) || 16; // fallback to 16px if can't get line height
+          const rootFontSize = parseFloat(computedStyle.fontSize);
+          baseFontSizeRef.current = rootFontSize;
         }
-
-        const maxCollapsedHeight = lines * lineHeightRef.current;
+        const maxCollapsedHeight = baseFontSizeRef.current * lines;
         const contentHeight = element.scrollHeight;
 
         setShowToggle(contentHeight > maxCollapsedHeight);

inspect_ai/_view/www/src/state/hooks.ts

@@ -132,6 +132,11 @@ export const useFilteredSamples = () => {
   const evalDescriptor = useEvalDescriptor();
   const sampleSummaries = useSampleSummaries();
   const filter = useStore((state) => state.log.filter);
+  const setFilterError = useStore((state) => state.logActions.setFilterError);
+  const clearFilterError = useStore(
+    (state) => state.logActions.clearFilterError,
+  );
+
   const epoch = useStore((state) => state.log.epoch);
   const sort = useStore((state) => state.log.sort);
   const samplesDescriptor = useSampleDescriptor();
@@ -139,10 +144,19 @@ export const useFilteredSamples = () => {
 
   return useMemo(() => {
     // Apply filters
+    const { result, error, allErrors } =
+      evalDescriptor && filter
+        ? filterSamples(evalDescriptor, sampleSummaries, filter)
+        : { result: sampleSummaries, error: undefined, allErrors: false };
+
+    if (error && allErrors) {
+      setFilterError(error);
+    } else {
+      clearFilterError();
+    }
+
     const prefiltered =
-      evalDescriptor && filter.value
-        ? filterSamples(evalDescriptor, sampleSummaries, filter.value).result
-        : sampleSummaries;
+      error === undefined || !allErrors ? result : sampleSummaries;
 
     // Filter epochs
     const filtered =
@@ -160,6 +174,8 @@ export const useFilteredSamples = () => {
     evalDescriptor,
     sampleSummaries,
     filter,
+    setFilterError,
+    clearFilterError,
     epoch,
     sort,
     samplesDescriptor,

inspect_ai/_view/www/src/state/logSlice.ts

@@ -1,4 +1,4 @@
-import { LogState, ScoreFilter, ScoreLabel } from "../app/types";
+import { FilterError, LogState, ScoreLabel } from "../app/types";
 import { EvalSummary, PendingSamples } from "../client/api/types";
 import { kDefaultSort, kLogViewInfoTabId } from "../constants";
 import { createLogger } from "../utils/logger";
@@ -23,7 +23,13 @@ export interface LogSlice {
     setPendingSampleSummaries: (samples: PendingSamples) => void;
 
     // Set filter criteria
-    setFilter: (filter: ScoreFilter) => void;
+    setFilter: (filter: string) => void;
+
+    // Set the filter error
+    setFilterError: (error: FilterError) => void;
+
+    // Clear the filter error
+    clearFilterError: () => void;
 
     // Set epoch filter
     setEpoch: (epoch: string) => void;
@@ -60,7 +66,9 @@ const initialState = {
   loadedLog: undefined,
 
   // Filter state
-  filter: {},
+  filter: "",
+  filterError: undefined,
+
   epoch: "all",
   sort: kDefaultSort,
   score: undefined,
@@ -110,10 +118,19 @@ export const createLogSlice = (
           state.log.pendingSampleSummaries = pendingSampleSummaries;
         }),
 
-      setFilter: (filter: ScoreFilter) =>
+      setFilter: (filter: string) =>
         set((state) => {
           state.log.filter = filter;
         }),
+      setFilterError: (error: FilterError) =>
+        set((state) => {
+          state.log.filterError = error;
+        }),
+      clearFilterError: () => {
+        set((state) => {
+          state.log.filterError = undefined;
+        });
+      },
       setEpoch: (epoch: string) =>
         set((state) => {
           state.log.epoch = epoch;
@@ -132,7 +149,8 @@ export const createLogSlice = (
         }),
       resetFiltering: () =>
         set((state) => {
-          state.log.filter = {};
+          state.log.filter = "";
+          state.log.filterError = undefined;
           state.log.epoch = "all";
           state.log.sort = kDefaultSort;
           state.log.score = undefined;

inspect_ai/_view/www/yarn.lock

@@ -4254,17 +4254,17 @@ react-refresh@^0.17.0:
   resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.17.0.tgz#b7e579c3657f23d04eccbe4ad2e58a8ed51e7e53"
   integrity sha512-z6F7K9bV85EfseRCp2bzrpyQ0Gkw1uLoCel9XBVWPg/TjRj94SkJzUTGfOa4bs7iJvBWtQG0Wq7wnI0syw3EBQ==
 
-react-router-dom@^7.5.0:
-  version "7.5.1"
-  resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.5.1.tgz#16ffa63006dfdbab53cf243be84c29535e7dc4e0"
-  integrity sha512-5DPSPc7ENrt2tlKPq0FtpG80ZbqA9aIKEyqX6hSNJDlol/tr6iqCK4crqdsusmOSSotq6zDsn0y3urX9TuTNmA==
+react-router-dom@^7.5.3:
+  version "7.5.3"
+  resolved "https://registry.yarnpkg.com/react-router-dom/-/react-router-dom-7.5.3.tgz#496e9f6d90f731703c7772668b41747028e0a2d5"
+  integrity sha512-cK0jSaTyW4jV9SRKAItMIQfWZ/D6WEZafgHuuCb9g+SjhLolY78qc+De4w/Cz9ybjvLzShAmaIMEXt8iF1Cm+A==
   dependencies:
-    react-router "7.5.1"
+    react-router "7.5.3"
 
-react-router@7.5.1:
-  version "7.5.1"
-  resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.5.1.tgz#e0bae54e913d139e727e7a88d165174a2cdbeb27"
-  integrity sha512-/jjU3fcYNd2bwz9Q0xt5TwyiyoO8XjSEFXJY4O/lMAlkGTHWuHRAbR9Etik+lSDqMC7A7mz3UlXzgYT6Vl58sA==
+react-router@7.5.3:
+  version "7.5.3"
+  resolved "https://registry.yarnpkg.com/react-router/-/react-router-7.5.3.tgz#9e5420832af8c3690740c1797d4fa54613fea06d"
+  integrity sha512-3iUDM4/fZCQ89SXlDa+Ph3MevBrozBAI655OAfWQlTm9nBR0IKlrmNwFow5lPHttbwvITZfkeeeZFP6zt3F7pw==
   dependencies:
     cookie "^1.0.1"
     set-cookie-parser "^2.6.0"

inspect_ai/agent/_as_solver.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 from typing import TYPE_CHECKING, Any
 
 from inspect_ai.util._limit import Limit, apply_limits
+from inspect_ai.util._span import span
 
 if TYPE_CHECKING:
     from inspect_ai.solver._solver import Solver
@@ -61,7 +62,8 @@ def as_solver(agent: Agent, limits: list[Limit] = [], **agent_kwargs: Any) -> So
             try:
                 # run the agent with limits
                 with apply_limits(limits):
-                    agent_state = await agent(agent_state, **agent_kwargs)
+                    async with span(name=agent_name, type="agent"):
+                        agent_state = await agent(agent_state, **agent_kwargs)
             # if an exception occurs, we still want to update the TaskState with the
             # AgentState's messages + output so that it appears in the log and is scored
             finally: