inspect-ai 0.3.99__py3-none-any.whl → 0.3.101__py3-none-any.whl

inspect_ai/_view/www/src/app/samples/transcript/outline/OutlineRow.tsx

@@ -0,0 +1,223 @@
+import clsx from "clsx";
+import { FC, ReactNode, useRef } from "react";
+import { Link } from "react-router-dom";
+import { PopOver } from "../../../../components/PopOver";
+import { PulsingDots } from "../../../../components/PulsingDots";
+import {
+  useCollapseSampleEvent,
+  useSamplePopover,
+} from "../../../../state/hooks";
+import { formatDateTime, formatTime } from "../../../../utils/format";
+import { parsePackageName } from "../../../../utils/python";
+import { ApplicationIcons } from "../../../appearance/icons";
+import { MetaDataGrid } from "../../../content/MetaDataGrid";
+import { useSampleEventUrl } from "../../../routing/url";
+import { kSandboxSignalName } from "../transform/fixups";
+import { EventNode } from "../types";
+import styles from "./OutlineRow.module.css";
+
+export interface OutlineRowProps {
+  node: EventNode;
+  collapseScope: string;
+  running?: boolean;
+  selected?: boolean;
+}
+
+export const OutlineRow: FC<OutlineRowProps> = ({
+  node,
+  collapseScope,
+  running,
+  selected,
+}) => {
+  const [collapsed, setCollapsed] = useCollapseSampleEvent(
+    collapseScope,
+    node.id,
+  );
+  const icon = iconForNode(node);
+  const toggle = toggleIcon(node, collapsed);
+
+  const popoverId = `${node.id}-popover`;
+  const { isShowing } = useSamplePopover(popoverId);
+
+  const ref = useRef(null);
+
+  // Get all URL parameters at component level
+  const sampleEventUrl = useSampleEventUrl(node.id);
+
+  return (
+    <>
+      <div
+        className={clsx(
+          styles.eventRow,
+          "text-size-smallest",
+          selected ? styles.selected : "",
+        )}
+        style={{ paddingLeft: `${node.depth * 0.4}em` }}
+      >
+        <div
+          className={clsx(styles.toggle)}
+          onClick={() => {
+            setCollapsed(!collapsed);
+          }}
+        >
+          {toggle ? <i className={clsx(toggle)} /> : undefined}
+        </div>
+        <div className={clsx(styles.label)} data-depth={node.depth}>
+          {icon ? <i className={clsx(icon, styles.icon)} /> : undefined}
+          {sampleEventUrl ? (
+            <Link
+              to={sampleEventUrl}
+              className={clsx(styles.eventLink)}
+              ref={ref}
+            >
+              {parsePackageName(labelForNode(node)).module}
+            </Link>
+          ) : (
+            <span ref={ref}>{parsePackageName(labelForNode(node)).module}</span>
+          )}
+          {running ? (
+            <PulsingDots
+              size="small"
+              className={clsx(styles.progress)}
+              subtle={false}
+            />
+          ) : undefined}
+        </div>
+      </div>
+      <PopOver
+        id={`${node.id}-popover`}
+        positionEl={ref.current}
+        isOpen={isShowing}
+        className={clsx(styles.popper)}
+        placement="auto-end"
+      >
+        {summarizeNode(node)}
+      </PopOver>
+    </>
+  );
+};
+
+const toggleIcon = (
+  node: EventNode,
+  collapsed: boolean,
+): string | undefined => {
+  if (node.children.length > 0) {
+    return collapsed
+      ? ApplicationIcons.chevron.right
+      : ApplicationIcons.chevron.down;
+  }
+};
+
+const iconForNode = (node: EventNode): string | undefined => {
+  switch (node.event.event) {
+    case "sample_limit":
+      return ApplicationIcons.limits.custom;
+
+    case "score":
+      return ApplicationIcons.scorer;
+
+    case "error":
+      return ApplicationIcons.error;
+  }
+};
+
+const labelForNode = (node: EventNode): string => {
+  if (node.event.event === "span_begin") {
+    switch (node.event.type) {
+      case "solver":
+        return node.event.name;
+      case "tool":
+        return node.event.name;
+      default: {
+        if (node.event.name === kSandboxSignalName) {
+          return "sandbox events";
+        }
+        return node.event.name;
+      }
+    }
+  } else {
+    switch (node.event.event) {
+      case "subtask":
+        return node.event.name;
+      case "approval":
+        switch (node.event.decision) {
+          case "approve":
+            return "approved";
+          case "reject":
+            return "rejected";
+          case "escalate":
+            return "escalated";
+          case "modify":
+            return "modified";
+          case "terminate":
+            return "terminated";
+          default:
+            return node.event.decision;
+        }
+      case "model":
+        return `model${node.event.role ? ` (${node.event.role})` : ""}`;
+      case "score":
+        return "scoring";
+      case "step":
+        if (node.event.name === kSandboxSignalName) {
+          return "sandbox events";
+        }
+        return node.event.name;
+
+      default:
+        return node.event.event;
+    }
+  }
+};
+
+export const summarizeNode = (node: EventNode): ReactNode => {
+  let entries: Record<string, unknown> = {};
+  switch (node.event.event) {
+    case "sample_init":
+      entries = {
+        sample_id: node.event.sample.id,
+        sandbox: node.event.sample.sandbox?.type,
+        started: formatDateTime(new Date(node.event.timestamp)),
+        working_start: formatTime(node.event.working_start),
+      };
+      break;
+
+    case "sample_limit":
+      entries = {
+        type: node.event.type,
+        message: node.event.message,
+        limit: node.event.limit,
+        started: formatDateTime(new Date(node.event.timestamp)),
+        working_start: formatTime(node.event.working_start),
+      };
+      break;
+    case "score":
+      entries = {
+        answer: node.event.score.answer,
+        score: node.event.score.value,
+        started: formatDateTime(new Date(node.event.timestamp)),
+        working_start: formatTime(node.event.working_start),
+      };
+      break;
+    case "span_begin":
+      entries = {
+        name: node.event.name,
+        started: formatDateTime(new Date(node.event.timestamp)),
+        working_start: formatTime(node.event.working_start),
+      };
+      break;
+    default:
+      entries = {
+        started: formatDateTime(new Date(node.event.timestamp)),
+        working_start: formatTime(node.event.working_start),
+      };
+  }
+
+  return (
+    <MetaDataGrid
+      entries={entries}
+      size="mini"
+      className={clsx(styles.popover, "text-size-smallest")}
+    />
+  );
+};

inspect_ai/_view/www/src/app/samples/transcript/outline/TranscriptOutline.module.css

@@ -0,0 +1,10 @@
+.node {
+  display: grid;
+  column-gap: 0.3em;
+  grid-template-columns: max-content 1fr;
+}
+
+.panel {
+  margin-top: 0.65rem;
+  overflow: visible;
+}

inspect_ai/_view/www/src/app/samples/transcript/outline/TranscriptOutline.tsx

@@ -0,0 +1,258 @@
+import {
+  CSSProperties,
+  FC,
+  RefObject,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+} from "react";
+
+import { EventNode } from "../types";
+
+import clsx from "clsx";
+import { Virtuoso, VirtuosoHandle } from "react-virtuoso";
+import { useScrollTrack, useVirtuosoState } from "../../../../state/scrolling";
+import { useStore } from "../../../../state/store";
+import { flatTree } from "../transform/treeify";
+
+import { useSampleDetailNavigation } from "../../../routing/navigationHooks";
+import { kSandboxSignalName } from "../transform/fixups";
+import { OutlineRow } from "./OutlineRow";
+import styles from "./TranscriptOutline.module.css";
+import {
+  collapseScoring,
+  collapseTurns,
+  makeTurns,
+  noScorerChildren,
+  removeNodeVisitor,
+  removeStepSpanNameVisitor,
+} from "./tree-visitors";
+
+const kCollapseScope = "transcript-outline";
+const kFramesToStabilize = 10;
+
+interface TranscriptOutlineProps {
+  eventNodes: EventNode[];
+  defaultCollapsedIds: Record<string, boolean>;
+  running?: boolean;
+  className?: string | string[];
+  scrollRef?: RefObject<HTMLDivElement | null>;
+  style?: CSSProperties;
+}
+
+// hack: add a padding node to the end of the list so
+// when the tree is positioned at the bottom of the viewport
+// it has some breathing room
+const EventPaddingNode: EventNode = {
+  id: "padding",
+  event: {
+    event: "info",
+    source: "",
+    data: "",
+    timestamp: "",
+    pending: false,
+    working_start: 0,
+    span_id: null,
+  },
+  depth: 0,
+  children: [],
+};
+
+export const TranscriptOutline: FC<TranscriptOutlineProps> = ({
+  eventNodes,
+  defaultCollapsedIds,
+  running,
+  className,
+  scrollRef,
+  style,
+}) => {
+  const id = "transcript-tree";
+  // The virtual list handle and state
+  const listHandle = useRef<VirtuosoHandle | null>(null);
+  const { getRestoreState } = useVirtuosoState(listHandle, id);
+
+  // Collapse state
+  // The list of events that have been collapsed
+  const collapsedEvents = useStore((state) => state.sample.collapsedEvents);
+  const setCollapsedEvents = useStore(
+    (state) => state.sampleActions.setCollapsedEvents,
+  );
+
+  const selectedOutlineId = useStore((state) => state.sample.selectedOutlineId);
+  const setSelectedOutlineId = useStore(
+    (state) => state.sampleActions.setSelectedOutlineId,
+  );
+  const sampleDetailNavigation = useSampleDetailNavigation();
+
+  // Flag to indicate programmatic scrolling is in progress
+  const isProgrammaticScrolling = useRef(false);
+  // Last position to check for scroll stabilization
+  const lastScrollPosition = useRef<number | null>(null);
+  // Frame count for detecting scroll stabilization
+  const stableFrameCount = useRef(0);
+
+  useEffect(() => {
+    if (sampleDetailNavigation.event) {
+      // Set the flag to indicate we're in programmatic scrolling
+      isProgrammaticScrolling.current = true;
+      lastScrollPosition.current = null;
+      stableFrameCount.current = 0;
+
+      setSelectedOutlineId(sampleDetailNavigation.event);
+
+      // Start monitoring to detect when scrolling has stabilized
+      const checkScrollStabilized = () => {
+        if (!isProgrammaticScrolling.current) return;
+
+        const currentPosition = scrollRef?.current?.scrollTop ?? null;
+
+        if (currentPosition === lastScrollPosition.current) {
+          stableFrameCount.current++;
+
+          // If position has been stable for a few frames, consider scrolling complete
+          if (stableFrameCount.current >= kFramesToStabilize) {
+            isProgrammaticScrolling.current = false;
+            return;
+          }
+        } else {
+          // Reset stability counter if position changed
+          stableFrameCount.current = 0;
+          lastScrollPosition.current = currentPosition;
+        }
+
+        // Continue checking until scrolling stabilizes
+        requestAnimationFrame(checkScrollStabilized);
+      };
+
+      // Start the RAF loop to detect scroll stabilization
+      requestAnimationFrame(checkScrollStabilized);
+    }
+  }, [sampleDetailNavigation.event, setSelectedOutlineId, scrollRef]);
+
+  const outlineNodeList = useMemo(() => {
+    // flattten the event tree
+    const nodeList = flatTree(
+      eventNodes,
+      (collapsedEvents ? collapsedEvents[kCollapseScope] : undefined) ||
+        defaultCollapsedIds,
+      [
+        // Strip specific nodes
+        removeNodeVisitor("logger"),
+        removeNodeVisitor("info"),
+        removeNodeVisitor("state"),
+        removeNodeVisitor("store"),
+        removeNodeVisitor("approval"),
+        removeNodeVisitor("input"),
+        removeNodeVisitor("sandbox"),
+
+        // Strip the sandbox wrapper (and children)
+        removeStepSpanNameVisitor(kSandboxSignalName),
+
+        // Remove child events for scorers
+        noScorerChildren(),
+      ],
+    );
+
+    return collapseScoring(collapseTurns(makeTurns(nodeList)));
+  }, [eventNodes, collapsedEvents, defaultCollapsedIds]);
+
+  // Event node, for scroll tracking
+  const allNodesList = useMemo(() => {
+    return flatTree(eventNodes, null);
+  }, [eventNodes]);
+
+  const elementIds = allNodesList.map((node) => node.id);
+  const findNearestOutlineAbove = useCallback(
+    (targetId: string): EventNode | null => {
+      const targetIndex = allNodesList.findIndex(
+        (node) => node.id === targetId,
+      );
+      if (targetIndex === -1) return null;
+
+      const outlineIds = new Set(outlineNodeList.map((node) => node.id));
+
+      // Search backwards from target position (inclusive)
+      for (let i = targetIndex; i >= 0; i--) {
+        if (outlineIds.has(allNodesList[i].id)) {
+          return allNodesList[i];
+        }
+      }
+
+      return null;
+    },
+    [allNodesList, outlineNodeList],
+  );
+
+  useScrollTrack(
+    elementIds,
+    (id: string) => {
+      if (!isProgrammaticScrolling.current) {
+        // If the ID is not in the list, return
+        const parentNode = findNearestOutlineAbove(id);
+        if (parentNode) {
+          setSelectedOutlineId(parentNode.id);
+        }
+      }
+    },
+    scrollRef,
+  );
+
+  // Update the collapsed events when the default collapsed IDs change
+  // This effect only depends on defaultCollapsedIds, not eventNodes
+  useEffect(() => {
+    // Only initialize collapsedEvents if it's empty
+    if (!collapsedEvents && Object.keys(defaultCollapsedIds).length > 0) {
+      setCollapsedEvents(kCollapseScope, defaultCollapsedIds);
+    }
+  }, [defaultCollapsedIds, collapsedEvents, setCollapsedEvents]);
+
+  const renderRow = useCallback(
+    (index: number, node: EventNode) => {
+      if (node === EventPaddingNode) {
+        return (
+          <div
+            className={styles.eventPadding}
+            key={node.id}
+            style={{ height: "2em" }}
+          ></div>
+        );
+      } else {
+        return (
+          <OutlineRow
+            collapseScope={kCollapseScope}
+            node={node}
+            key={node.id}
+            running={running && index === outlineNodeList.length - 1}
+            selected={
+              selectedOutlineId ? selectedOutlineId === node.id : index === 0
+            }
+          />
+        );
+      }
+    },
+    [outlineNodeList, running, selectedOutlineId],
+  );
+
+  return (
+    <Virtuoso
+      ref={listHandle}
+      customScrollParent={scrollRef?.current ? scrollRef.current : undefined}
+      id={id}
+      style={{ ...style }}
+      data={[...outlineNodeList, EventPaddingNode]}
+      defaultItemHeight={50}
+      itemContent={renderRow}
+      atBottomThreshold={30}
+      increaseViewportBy={{ top: 300, bottom: 300 }}
+      overscan={{
+        main: 10,
+        reverse: 10,
+      }}
+      className={clsx(className, "transcript-outline")}
+      skipAnimationFrameInResizeObserver={true}
+      restoreStateFrom={getRestoreState()}
+      tabIndex={0}
+    />
+  );
+};

inspect_ai/_view/www/src/app/samples/transcript/outline/tree-visitors.ts

@@ -0,0 +1,187 @@
+import { ScoreEvent, SpanBeginEvent } from "../../../../@types/log";
+import { TYPE_SCORER, TYPE_SCORERS } from "../transform/utils";
+import { EventNode } from "../types";
+
+const kTurnType = "turn";
+const kTurnsType = "turns";
+const kCollapsedScoring = "scorings";
+
+// Visitors are used to transform the event tree
+export const removeNodeVisitor = (event: string) => {
+  return {
+    visit: (node: EventNode): EventNode[] => {
+      if (node.event.event === event) {
+        return [];
+      }
+      return [node];
+    },
+  };
+};
+
+export const removeStepSpanNameVisitor = (name: string) => {
+  return {
+    visit: (node: EventNode): EventNode[] => {
+      if (
+        (node.event.event === "step" || node.event.event === "span_begin") &&
+        node.event.name === name
+      ) {
+        return [];
+      }
+      return [node];
+    },
+  };
+};
+
+export const noScorerChildren = () => {
+  let inScorers = false;
+  let inScorer = false;
+  let currentDepth = -1;
+  return {
+    visit: (node: EventNode): EventNode[] => {
+      // Note once we're in the scorers span
+      if (
+        node.event.event === "span_begin" &&
+        node.event.type === TYPE_SCORERS
+      ) {
+        inScorers = true;
+        return [node];
+      }
+
+      if (
+        (node.event.event === "step" || node.event.event === "span_begin") &&
+        node.event.type === TYPE_SCORER
+      ) {
+        inScorer = true;
+        currentDepth = node.depth;
+        return [node];
+      }
+
+      if (inScorers && inScorer && node.depth === currentDepth + 1) {
+        return [];
+      }
+      return [node];
+    },
+  };
+};
+
+export const makeTurns = (eventNodes: EventNode[]): EventNode[] => {
+  const results: EventNode[] = [];
+  let modelNode: EventNode | null = null;
+  const toolNodes: EventNode[] = [];
+  let turnCount = 1;
+
+  const makeTurn = (force?: boolean) => {
+    if (modelNode !== null && (force || toolNodes.length > 0)) {
+      // Create a new "turn" node based on the model event
+      const turnNode = new EventNode(
+        modelNode.id,
+        {
+          id: modelNode.id,
+          event: "span_begin",
+          type: kTurnType,
+          name: `turn ${turnCount++}`,
+          pending: false,
+          working_start: modelNode.event.working_start,
+          timestamp: modelNode.event.timestamp,
+          parent_id: null,
+          span_id: modelNode.event.span_id,
+        },
+        modelNode.depth,
+      );
+
+      // Add the original model event and tool events as children
+      turnNode.children = [modelNode, ...toolNodes];
+      results.push(turnNode);
+    }
+    modelNode = null;
+    toolNodes.length = 0;
+  };
+
+  for (const node of eventNodes) {
+    if (node.event.event === "model") {
+      if (modelNode !== null && toolNodes.length === 0) {
+        // back to back model calls are considered a single turn
+        makeTurn(true);
+      } else {
+        makeTurn();
+        modelNode = node;
+      }
+    } else if (node.event.event === "tool") {
+      toolNodes.push(node);
+    } else {
+      makeTurn();
+      results.push(node);
+    }
+  }
+  makeTurn();
+
+  return results;
+};
+
+export const collapseTurns = (eventNodes: EventNode[]): EventNode[] => {
+  const results: EventNode[] = [];
+  const collecting: EventNode[] = [];
+  const collect = () => {
+    if (collecting.length > 0) {
+      const numberOfTurns = collecting.length;
+      const firstTurn = collecting[0];
+      const turnNode = new EventNode(
+        firstTurn.id,
+        {
+          ...(firstTurn.event as SpanBeginEvent),
+          name: `${numberOfTurns} ${numberOfTurns === 1 ? "turn" : "turns"}`,
+          type: kTurnsType,
+        },
+        firstTurn.depth,
+      );
+      results.push(turnNode);
+      collecting.length = 0;
+    }
+  };
+
+  for (const node of eventNodes) {
+    if (node.event.event === "span_begin" && node.event.type === kTurnType) {
+      collecting.push(node);
+    } else {
+      collect();
+      results.push(node);
+    }
+  }
+  // Handle any remaining collected turns
+  collect();
+  return results;
+};
+
+export const collapseScoring = (eventNodes: EventNode[]): EventNode[] => {
+  const results: EventNode[] = [];
+  const collecting: EventNode[] = [];
+  const collect = () => {
+    if (collecting.length > 0) {
+      const firstScore = collecting[0];
+      const turnNode = new EventNode(
+        firstScore.id,
+        {
+          ...(firstScore.event as ScoreEvent),
+          name: "scoring",
+          type: kCollapsedScoring,
+        },
+        firstScore.depth,
+      );
+      results.push(turnNode);
+      collecting.length = 0;
+    }
+  };
+
+  for (const node of eventNodes) {
+    if (node.event.event === "score") {
+      collecting.push(node);
+    } else {
+      collect();
+      results.push(node);
+    }
+  }
+
+  // Handle any remaining collected turns
+  collect();
+  return results;
+};

inspect_ai/_view/www/src/app/samples/transcript/state/StateEventRenderers.tsx

@@ -35,6 +35,9 @@ const system_msg_added_sig: ChangeType = {
   render: (_changes, resolvedState) => {
     const messages = resolvedState["messages"] as Array<unknown>;
     const message = messages[0];
+    if (typeof message !== "object" || !message) {
+      return <></>;
+    }
     return (
       <ChatView
         key="system_msg_event_preview"
@@ -172,7 +175,11 @@ const renderTools = (
     return change.path.startsWith("/tool_choice");
   });
   if (resolvedState.tool_choice && hasToolChoice) {
-    toolsInfo["Tool Choice"] = toolName(resolvedState.tool_choice);
+    toolsInfo["Tool Choice"] = (
+      <span className={clsx("text-size-smaller")}>
+        {toolName(resolvedState.tool_choice)}
+      </span>
+    );
   }
 
   // Show either all tools or just the specific tools