inspect-ai 0.3.57__py3-none-any.whl → 0.3.59__py3-none-any.whl

inspect_ai/_view/www/src/components/VirtualList.mjs

@@ -1,98 +1,280 @@
-// preact-virtual-list.mjs
-
-import { html, Component } from "htm/preact";
-import { createRef } from "preact";
+import { html } from "htm/preact";
+import { useRef, useState, useEffect, useMemo } from "preact/hooks";
+import { forwardRef, useImperativeHandle } from "preact/compat";
 import { throttle } from "../utils/sync.mjs";
 
-const STYLE_INNER =
-  "position:relative; overflow:hidden; width:100%; min-height:100%;";
-const STYLE_CONTENT =
-  "position:absolute; top:0; left:0; height:100%; width:100%; overflow:visible;";
+/**
+ * A virtualized list component that efficiently renders large lists by only
+ * rendering the items that are currently visible in the viewport.
+ * Supports dynamic row heights that are measured after rendering.
+ *
+ * @template T
+ * @param {Object} props - The component props
+ * @param {T[]} props.data - Array of items to be rendered in the list
+ * @param {(item: T, index: number) => preact.VNode} props.renderRow - Function to render each row
+ * @param {number} [props.overscanCount=15] - Number of extra rows to render above and below the visible area
+ * @param {number} [props.estimatedRowHeight=50] - Estimated height of each row before measurement
+ * @param {boolean} [props.sync=false] - If true, forces a re-render on scroll
+ * @param {import("preact").RefObject<HTMLElement>} [props.scrollRef] - Optional ref for the scroll container
+ * @param {import("preact").Ref<{ scrollToIndex: (index: number) => void }>} ref - Ref object exposing the list's methods
+ * @returns {preact.VNode} The virtualized list component
+ */
+export const VirtualList = forwardRef(
+  (
+    /** @type {props} */ {
+      data,
+      renderRow,
+      overscanCount = 15,
+      estimatedRowHeight = 50,
+      sync = false,
+      scrollRef,
+      ...props
+    },
+    ref,
+  ) => {
+    const [height, setHeight] = useState(0);
+    const [offset, setOffset] = useState(0);
+
+    const [listMetrics, setListMetrics] = useState({
+      rowHeights: new Map(),
+      totalHeight: data.length * estimatedRowHeight,
+    });
+
+    const baseRef = useRef(null);
+    const containerRef = useRef(null);
+    const rowRefs = useRef(new Map());
+
+    // Function to get row height (measured or estimated)
+    const getRowHeight = (index) => {
+      return listMetrics.rowHeights.get(index) || estimatedRowHeight;
+    };
+
+    // Calculate row positions based on current heights
+    const rowPositions = useMemo(() => {
+      let currentPosition = 0;
+      const positions = new Map();
+
+      for (let i = 0; i < data.length; i++) {
+        positions.set(i, currentPosition);
+        currentPosition += getRowHeight(i);
+      }
+
+      return positions;
+    }, [listMetrics.rowHeights, data.length]);
+
+    // Expose scrollToIndex method via ref
+    useImperativeHandle(
+      ref,
+      () => ({
+        focus: () => {
+          baseRef.current;
+        },
+        scrollToIndex: (index, direction) => {
+          const scrollElement = scrollRef?.current || baseRef.current;
+          if (!scrollElement || index < 0 || index >= data.length) return;
 
-export class VirtualList extends Component {
-  /** @type {HTMLElement} */ base;
+          const currentScrollTop = scrollElement.scrollTop;
+          const viewportHeight = scrollElement.offsetHeight;
 
-  constructor(props) {
-    super(props);
-    this.state = {
-      height: 0,
-      offset: 0,
+          // Get position and height of target row
+          const rowTop = rowPositions.get(index) || 0;
+          const rowHeight = getRowHeight(index);
+          const rowBottom = rowTop + rowHeight;
+
+          // If this is already visible, don't scroll
+          const isVisible =
+            rowTop >= currentScrollTop &&
+            rowBottom <= currentScrollTop + viewportHeight;
+          if (isVisible) {
+            return;
+          }
+
+          // Calculate new scroll position based on direction
+          let newScrollTop;
+          if (direction === "up") {
+            // Align top of element with top of viewport
+            newScrollTop = rowTop;
+          } else {
+            // Align bottom of element with bottom of viewport
+            newScrollTop = rowBottom - viewportHeight;
+          }
+
+          // Clamp scroll position to valid range
+          newScrollTop = Math.max(
+            0,
+            Math.min(newScrollTop, listMetrics.totalHeight - viewportHeight),
+          );
+          scrollElement.scrollTop = newScrollTop;
+        },
+      }),
+      [rowPositions, data.length],
+    );
+
+    // Measure rendered rows and update heights if needed
+    const measureRows = () => {
+      // Keep track of updated heights
+      let updates = [];
+
+      rowRefs.current.forEach((element, index) => {
+        if (element) {
+          const measuredHeight = element.offsetHeight;
+          // If the measured height is different, schedule an update
+          if (
+            measuredHeight &&
+            measuredHeight !== listMetrics.rowHeights.get(index)
+          ) {
+            updates.push([index, measuredHeight]);
+          }
+        }
+      });
+
+      // If no rows changed, do nothing
+      if (updates.length === 0) return;
+
+      // Create a new Map of rowHeights so we don't mutate state directly
+      const newHeights = new Map(listMetrics.rowHeights);
+      updates.forEach(([index, height]) => {
+        newHeights.set(index, height);
+      });
+
+      // Recompute total height only once
+      let newTotalHeight = 0;
+      for (let i = 0; i < data.length; i++) {
+        newTotalHeight += newHeights.get(i) || estimatedRowHeight;
+      }
+
+      // Now update our single state object in one go:
+      setListMetrics({
+        rowHeights: newHeights,
+        totalHeight: newTotalHeight,
+      });
     };
-    this.resize = () => {
-      if (this.state.height !== this.base.offsetHeight) {
-        this.setState({ height: this.base.offsetHeight });
+
+    // Handle container resize
+    const resize = () => {
+      const scrollElement = scrollRef?.current || baseRef.current;
+      if (scrollElement && height !== scrollElement.offsetHeight) {
+        setHeight(scrollElement.offsetHeight);
       }
     };
-    this.handleScroll = throttle(() => {
-      if (this.base) {
-        this.setState({ offset: this.base.scrollTop });
+
+    // Handle scroll with throttling
+    const handleScroll = throttle(() => {
+      const scrollElement = scrollRef?.current || baseRef.current;
+      if (scrollElement) {
+        setOffset(scrollElement.scrollTop);
       }
-      if (this.props.sync) {
-        this.forceUpdate();
+      if (sync) {
+        setOffset((prev) => prev);
       }
     }, 100);
-    this.containerRef = createRef();
-  }
-
-  componentDidUpdate() {
-    this.resize();
-  }
-
-  componentDidMount() {
-    this.resize();
-    window.addEventListener("resize", this.resize);
-  }
-
-  componentWillUnmount() {
-    window.removeEventListener("resize", this.resize);
-  }
-
-  render(
-    { data, rowMap, renderRow, overscanCount = 10, ...props },
-    { offset = 0, height = 0 },
-  ) {
-    // Compute the start and ending rows
-    const firstVisibleIdx = rowMap.findIndex((row) => {
-      return row.start + row.height >= offset;
-    });
-    const firstIndex = firstVisibleIdx > -1 ? firstVisibleIdx : 0;
 
-    const lastVisibleIdx = rowMap.findIndex((row) => {
-      return row.start + row.height >= offset + height;
+    // Setup scroll and resize listeners
+    useEffect(() => {
+      resize();
+      const scrollElement = scrollRef?.current || baseRef.current;
+
+      if (scrollElement) {
+        scrollElement.addEventListener("scroll", handleScroll);
+        window.addEventListener("resize", resize);
+
+        return () => {
+          scrollElement.removeEventListener("scroll", handleScroll);
+          window.removeEventListener("resize", resize);
+        };
+      }
+    }, [scrollRef?.current]);
+
+    // Measure rows after render
+    useEffect(() => {
+      measureRows();
     });
-    const lastIndex = lastVisibleIdx > -1 ? lastVisibleIdx : rowMap.length - 1;
-
-    // Compute the total height
-    const lastRow = rowMap[rowMap.length - 1];
-    const totalHeight = lastRow ? lastRow.start + lastRow.height : 0;
-
-    // Compute the visible rows (including overscan)
-    let visibleRowCount = lastIndex - firstIndex;
-    if (overscanCount) {
-      visibleRowCount += overscanCount;
-    }
-
-    // Account for overscan
-    const start = firstVisibleIdx;
-    const end = Math.min(data.length, start + visibleRowCount);
-
-    const selection = data.slice(start, end);
-
-    // const firstRow
-    const top = firstVisibleIdx !== -1 ? rowMap[firstVisibleIdx].start : 0;
-    const rows = html`<div onscroll=${this.handleScroll} ...${props}>
-      <div style=${`${STYLE_INNER} height:${totalHeight}px;`}>
-        <div style=${`${STYLE_CONTENT} top:${top}px;`} ref=${this.containerRef}>
-          ${selection.map((item, index) => {
-            const component = renderRow(item, start + index);
-
-            return html` <div key=${`list-item-${start + index}`}>
-              ${component}
-            </div>`;
-          })}
+
+    const findRowAtOffset = (targetOffset) => {
+      if (targetOffset <= 0) return 0;
+      if (targetOffset >= listMetrics.totalHeight) return data.length - 1;
+
+      let low = 0;
+      let high = data.length - 1;
+      let lastValid = 0;
+
+      while (low <= high) {
+        const mid = Math.floor((low + high) / 2);
+        const rowStart = rowPositions.get(mid) || 0;
+
+        if (rowStart <= targetOffset) {
+          lastValid = mid;
+          low = mid + 1;
+        } else {
+          high = mid - 1;
+        }
+      }
+      return lastValid;
+    };
+
+    const firstVisibleIdx = findRowAtOffset(offset);
+    const lastVisibleIdx = findRowAtOffset(offset + height);
+
+    // Calculate range of rows to render including overscan
+    const start = Math.max(0, firstVisibleIdx - overscanCount);
+    const end = Math.min(data.length, lastVisibleIdx + overscanCount);
+
+    // Memoize the rendered rows to prevent unnecessary re-renders
+    const renderedRows = useMemo(() => {
+      const selection = data.slice(start, end);
+      return selection.map((item, index) => {
+        const actualIndex = start + index;
+        return html`
+          <div
+            key=${`list-item-${actualIndex}`}
+            ref=${(el) => {
+              if (el) {
+                rowRefs.current.set(actualIndex, el);
+              } else {
+                rowRefs.current.delete(actualIndex);
+              }
+            }}
+          >
+            ${renderRow(item, actualIndex)}
+          </div>
+        `;
+      });
+    }, [data, start, end, renderRow]);
+
+    const style_inner = {
+      position: "relative",
+      overflow: scrollRef?.current ? "visible" : "hidden",
+      width: "100%",
+      minHeight: "100%",
+    };
+
+    const style_content = {
+      position: "absolute",
+      top: 0,
+      left: 0,
+      height: "100%",
+      width: "100%",
+      overflow: "visible",
+    };
+
+    const top = rowPositions.get(start) || 0;
+
+    // Only attach onscroll to baseRef if no scrollRef is provided
+    const scrollProps = scrollRef ? {} : { onscroll: handleScroll };
+
+    return html`
+      <div ref=${baseRef} ...${props} ...${scrollProps}>
+        <div
+          style=${{ ...style_inner, height: `${listMetrics.totalHeight}px` }}
+        >
+          <div
+            style=${{ ...style_content, top: `${top}px` }}
+            ref=${containerRef}
+          >
+            ${renderedRows}
+          </div>
         </div>
       </div>
-    </div>`;
-    return rows;
-  }
-}
+    `;
+  },
+);

inspect_ai/_view/www/src/index.js

@@ -10,7 +10,7 @@ import { throttle } from "./utils/sync.mjs";
 const vscode = getVscodeApi();
 let initialState = undefined;
 if (vscode) {
-  initialState = vscode.getState();
+  initialState = filterState(vscode.getState());
 }
 
 render(
@@ -20,9 +20,82 @@ render(
     saveInitialState=${throttle((state) => {
       const vscode = getVscodeApi();
       if (vscode) {
-        vscode.setState(state);
+        vscode.setState(filterState(state));
       }
     }, 1000)}
   />`,
   document.getElementById("app"),
 );
+
+function filterState(state) {
+  if (!state) {
+    return state;
+  }
+
+  // When saving state, we can't store vast amounts of data (like a large sample)
+  const filters = [filterLargeSample, filterLargeSelectedLog];
+  return filters.reduce(
+    (filteredState, filter) => filter(filteredState),
+    state,
+  );
+}
+
+// Filters the selected Sample if it is large
+function filterLargeSample(state) {
+  if (!state || !state.selectedSample) {
+    return state;
+  }
+
+  const estimatedTotalSize = estimateSize(state.selectedSample.messages);
+  if (estimatedTotalSize > 400000) {
+    const { selectedSample, ...filteredState } = state; // eslint-disable-line
+    return filteredState;
+  } else {
+    return state;
+  }
+}
+
+// Filters the selectedlog if it is too large
+function filterLargeSelectedLog(state) {
+  if (!state || !state.selectedLog?.contents) {
+    return state;
+  }
+
+  const estimatedSize = estimateSize(
+    state.selectedLog.contents.sampleSummaries,
+  );
+  if (estimatedSize > 400000) {
+    const { selectedLog, ...filteredState } = state; // eslint-disable-line
+    return filteredState;
+  } else {
+    return state;
+  }
+}
+
+function estimateSize(list, frequency = 0.2) {
+  if (!list || list.len === 0) {
+    return 0;
+  }
+
+  // Total number of samples
+  const sampleSize = Math.ceil(list.length * frequency);
+
+  // Get a proper random sample without duplicates
+  const messageIndices = new Set();
+  while (
+    messageIndices.size < sampleSize &&
+    messageIndices.size < list.length
+  ) {
+    const randomIndex = Math.floor(Math.random() * list.length);
+    messageIndices.add(randomIndex);
+  }
+
+  // Calculate size from sampled messages
+  const totalSize = Array.from(messageIndices).reduce((size, index) => {
+    return size + JSON.stringify(list[index]).length;
+  }, 0);
+
+  // Estimate total size based on sample
+  const estimatedTotalSize = (totalSize / sampleSize) * list.length;
+  return estimatedTotalSize;
+}

inspect_ai/_view/www/src/navbar/Navbar.mjs

@@ -19,6 +19,7 @@ import { SecondaryBar } from "./SecondaryBar.mjs";
  * @param {import("../types/log").EvalPlan} [props.evalPlan] - The EvalSpec
  * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats
  * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples
+ * @param {import("../samples/SamplesDescriptor.mjs").EvalDescriptor} [props.evalDescriptor] - The EvalDescriptor
  * @param {string} [props.status] - the status
  * @param {boolean} props.offcanvas - Are we in offcanvas mode?
  * @param {boolean} props.showToggle - Should we show the toggle?
@@ -32,6 +33,7 @@ export const Navbar = ({
   evalResults,
   evalStats,
   samples,
+  evalDescriptor,
   showToggle,
   offcanvas,
   status,
@@ -182,6 +184,7 @@ export const Navbar = ({
           evalResults=${evalResults}
           evalStats=${evalStats}
           samples=${samples}
+          evalDescriptor=${evalDescriptor}
           status=${status}
           style=${{ gridColumn: "1/-1" }}
         />

inspect_ai/_view/www/src/navbar/SecondaryBar.mjs

@@ -3,6 +3,7 @@ import { html } from "htm/preact";
 import { LabeledValue } from "../components/LabeledValue.mjs";
 import { formatDataset, formatDuration } from "../utils/Format.mjs";
 import { ExpandablePanel } from "../components/ExpandablePanel.mjs";
+import { scoreFilterItems } from "../samples/tools/filters.mjs";
 
 /**
  * Renders the Navbar
@@ -13,6 +14,7 @@ import { ExpandablePanel } from "../components/ExpandablePanel.mjs";
  * @param {import("../types/log").EvalResults} [props.evalResults] - The EvalResults
  * @param {import("../types/log").EvalStats} [props.evalStats] - The EvalStats
  * @param {import("../api/Types.mjs").SampleSummary[]} [props.samples] - the samples
+ * @param {import("../samples/SamplesDescriptor.mjs").EvalDescriptor} [props.evalDescriptor] - The EvalDescriptor
  * @param {string} [props.status] - the status
  * @param {Map<string, string>} [props.style] - is this off canvas
  *
@@ -24,6 +26,7 @@ export const SecondaryBar = ({
   evalResults,
   evalStats,
   samples,
+  evalDescriptor,
   status,
   style,
 }) => {
@@ -60,8 +63,8 @@ export const SecondaryBar = ({
   values.push({
     size: "minmax(12%, auto)",
     value: html`<${LabeledValue} label="${label}" style=${staticColStyle} style=${{ justifySelf: hasConfig ? "left" : "center" }}>
-    <${ScorerSummary} 
-      scorers=${evalResults?.scores} />
+    <${ScorerSummary}
+      evalDescriptor=${evalDescriptor} />
   </${LabeledValue}>`,
   });
 
@@ -124,17 +127,23 @@ const DatasetSummary = ({ dataset, samples, epochs, style }) => {
   `;
 };
 
-const ScorerSummary = ({ scorers }) => {
-  if (!scorers) {
+const ScorerSummary = ({ evalDescriptor }) => {
+  if (!evalDescriptor) {
     return "";
   }
 
-  const uniqScorers = new Set();
-  scorers.forEach((scorer) => {
-    uniqScorers.add(scorer.name);
-  });
+  const items = scoreFilterItems(evalDescriptor);
 
-  return Array.from(uniqScorers).join(", ");
+  return html`
+    <span style=${{ position: "relative" }}>
+      ${Array.from(items).map(
+        (item, index) => html`
+          ${index > 0 ? ", " : ""}
+          <span title=${item.tooltip}>${item.canonicalName}</span>
+        `,
+      )}
+    </span>
+  `;
 };
 
 /**

inspect_ai/_view/www/src/samples/SampleDialog.mjs

@@ -1,5 +1,5 @@
 import { html } from "htm/preact";
-import { useCallback, useMemo } from "preact/hooks";
+import { useCallback, useMemo, useRef } from "preact/hooks";
 
 import { ApplicationIcons } from "../appearance/Icons.mjs";
 import { LargeModal } from "../components/LargeModal.mjs";
@@ -43,6 +43,8 @@ export const SampleDialog = ({
   sampleScrollPositionRef,
   setSampleScrollPosition,
 }) => {
+  const scrollRef = useRef(/** @type {HTMLElement|null} */ (null));
+
   const tools = useMemo(() => {
     const nextTool = {
       label: "Next Sample",
@@ -94,6 +96,7 @@ export const SampleDialog = ({
           sampleDescriptor=${sampleDescriptor}
           selectedTab=${selectedTab}
           setSelectedTab=${setSelectedTab}
+          scrollRef=${scrollRef}
         />`;
   }, [id, sample, sampleDescriptor, selectedTab, setSelectedTab, sampleError]);
 
@@ -113,6 +116,7 @@ export const SampleDialog = ({
       showProgress=${sampleStatus === "loading"}
       initialScrollPositionRef=${sampleScrollPositionRef}
       setInitialScrollPosition=${setSampleScrollPosition}
+      scrollRef=${scrollRef}
     >
         ${children}
     </${LargeModal}>`;

inspect_ai/_view/www/src/samples/SampleDisplay.mjs

@@ -1,6 +1,6 @@
 import { html } from "htm/preact";
 
-import { ChatView } from "../components/ChatView.mjs";
+import { ChatViewVirtualList } from "../components/ChatView.mjs";
 import { MetaDataView } from "../components/MetaDataView.mjs";
 import { TabSet, TabPanel } from "../components/TabSet.mjs";
 
@@ -47,6 +47,7 @@ import {
  * @param {import("../samples/SamplesDescriptor.mjs").SamplesDescriptor} props.sampleDescriptor - the sample descriptor
  * @param {string} props.selectedTab - The selected tab
  * @param {(tab: string) => void} props.setSelectedTab - function to set the selected tab
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable element whic contains this display
  * @returns {import("preact").JSX.Element} The TranscriptView component.
  */
 export const InlineSampleDisplay = ({
@@ -57,6 +58,7 @@ export const InlineSampleDisplay = ({
   sampleDescriptor,
   selectedTab,
   setSelectedTab,
+  scrollRef,
 }) => {
   return html`<div style=${{ flexDirection: "row", width: "100%" }}>
     <${ProgressBar}
@@ -77,6 +79,7 @@ export const InlineSampleDisplay = ({
             sampleDescriptor=${sampleDescriptor}
             selectedTab=${selectedTab}
             setSelectedTab=${setSelectedTab}
+            scrollRef=${scrollRef}
           />`}
     </div>
   </div>`;
@@ -91,6 +94,7 @@ export const InlineSampleDisplay = ({
  * @param {import("../samples/SamplesDescriptor.mjs").SamplesDescriptor} props.sampleDescriptor - the sample descriptor
  * @param {string} props.selectedTab - The selected tab
  * @param {(tab: string) => void} props.setSelectedTab - function to set the selected tab
+ * @param {import("htm/preact").MutableRef<HTMLElement>} props.scrollRef - The scrollable parent element
  * @returns {import("preact").JSX.Element} The TranscriptView component.
  */
 export const SampleDisplay = ({
@@ -99,6 +103,7 @@ export const SampleDisplay = ({
   sampleDescriptor,
   selectedTab,
   setSelectedTab,
+  scrollRef,
 }) => {
   // Tab ids
   const baseId = `sample-dialog`;
@@ -120,13 +125,14 @@ export const SampleDisplay = ({
     html`
     <${TabPanel} id=${kSampleMessagesTabId} classes="sample-tab" title="Messages" onSelected=${onSelectedTab} selected=${
       selectedTab === kSampleMessagesTabId
-    }>
-      <${ChatView} 
+    } scrollable=${false} style=${{ width: "100%" }}>
+      <${ChatViewVirtualList} 
         key=${`${baseId}-chat-${id}`} 
         id=${`${baseId}-chat-${id}`} 
         messages=${sample.messages} 
-        style=${{ paddingLeft: ".8em", paddingTop: "1em" }}
+        style=${{ marginLeft: ".8em", marginTop: "1em" }}
         indented=${true}
+        scrollRef=${scrollRef}
       />
     </${TabPanel}>`,
   ];
@@ -136,7 +142,7 @@ export const SampleDisplay = ({
       <${TabPanel} id=${kSampleTranscriptTabId} classes="sample-tab" title="Transcript" onSelected=${onSelectedTab} selected=${
         selectedTab === kSampleTranscriptTabId || selectedTab === undefined
       } scrollable=${false}>
-        <${SampleTranscript} key=${`${baseId}-transcript-display-${id}`} id=${`${baseId}-transcript-display-${id}`} evalEvents=${sample.events}/>
+        <${SampleTranscript} key=${`${baseId}-transcript-display-${id}`} id=${`${baseId}-transcript-display-${id}`} evalEvents=${sample.events} scrollRef=${scrollRef}/>
       </${TabPanel}>`);
   }
 
@@ -201,16 +207,18 @@ export const SampleDisplay = ({
     );
   }
 
-  tabs.push(html`<${TabPanel} 
-          id=${kSampleJsonTabId} 
-          classes="sample-tab"
-          title="JSON" 
-          onSelected=${onSelectedTab} 
-          selected=${selectedTab === kSampleJsonTabId}>
-         <div style=${{ paddingLeft: "0.8em", marginTop: "0.4em" }}> 
-          <${JSONPanel} data=${sample} simple=${true}/>
-        </div>
-      </${TabPanel}>`);
+  if (sample.messages.length < 100) {
+    tabs.push(html`<${TabPanel} 
+        id=${kSampleJsonTabId} 
+        classes="sample-tab"
+        title="JSON" 
+        onSelected=${onSelectedTab} 
+        selected=${selectedTab === kSampleJsonTabId}>
+      <div style=${{ paddingLeft: "0.8em", marginTop: "0.4em" }}> 
+        <${JSONPanel} data=${sample} simple=${true}/>
+      </div>
+    </${TabPanel}>`);
+  }
 
   const tabsetId = `task-sample-details-tab-${id}`;
   const targetId = `${tabsetId}-content`;