inspect-ai 0.3.96__py3-none-any.whl → 0.3.97__py3-none-any.whl

inspect_ai/_view/www/src/state/sampleSlice.ts

@@ -4,20 +4,35 @@ import { SampleSummary } from "../client/api/types";
 import { kSampleMessagesTabId } from "../constants";
 import { createLogger } from "../utils/logger";
 import { createSamplePolling } from "./samplePolling";
-import { resolveSample } from "./sampleUtils"; // Import the shared utility
+import { resolveSample } from "./sampleUtils";
 import { StoreState } from "./store";
+import { isLargeSample } from "./store_filter";
 
 const log = createLogger("sampleSlice");
 
+// Create a module-level ref to store large sample objects
+let selectedSampleRef: { current: EvalSample | undefined } = {
+  current: undefined,
+};
+
 export interface SampleSlice {
   sample: SampleState;
   sampleActions: {
     // The actual sample data
     setSelectedSample: (sample: EvalSample) => void;
+    getSelectedSample: () => EvalSample | undefined;
     clearSelectedSample: () => void;
     setSampleStatus: (status: SampleStatus) => void;
     setSampleError: (error: Error | undefined) => void;
 
+    setCollapsedEvents: (collapsed: Record<string, true>) => void;
+    collapseEvent: (id: string, collapsed: boolean) => void;
+    clearCollapsedEvents: () => void;
+
+    setCollapsedIds: (key: string, collapsed: Record<string, true>) => void;
+    collapseId: (key: string, id: string, collapsed: boolean) => void;
+    clearCollapsedIds: (key: string) => void;
+
     // Loading
     loadSample: (
       logFile: string,
@@ -32,12 +47,23 @@ export interface SampleSlice {
 }
 
 const initialState: SampleState = {
-  selectedSample: undefined,
+  // Store ID for all samples (used for triggering renders)
+  sample_identifier: undefined,
+  // Store the actual sample object for small samples
+  selectedSampleObject: undefined,
+  // Flag to indicate where the sample is stored
+  sampleInState: false,
   sampleStatus: "ok",
   sampleError: undefined,
 
+  // signals that the sample needs to be reloaded
+  sampleNeedsReload: 0,
+
   // The resolved events
   runningEvents: [],
+  collapsedEvents: null,
+
+  collapsedIdBuckets: {},
 };
 
 export const createSampleSlice = (
@@ -53,18 +79,49 @@ export const createSampleSlice = (
     sample: initialState,
     sampleActions: {
       setSelectedSample: (sample: EvalSample) => {
+        const isLarge = isLargeSample(sample);
+
+        // Update state based on sample size
         set((state) => {
-          state.sample.selectedSample = sample;
+          state.sample.sample_identifier = {
+            id: sample.id,
+            epoch: sample.epoch,
+          };
+          state.sample.sampleInState = !isLarge;
+
+          // Only store in state if it's small
+          if (!isLarge) {
+            state.sample.selectedSampleObject = sample;
+            // Clear ref if using state
+            selectedSampleRef.current = undefined;
+          } else {
+            // Use ref for large objects
+            state.sample.selectedSampleObject = undefined;
+            selectedSampleRef.current = sample;
+          }
         });
+
         if (sample.events.length < 1) {
           // If there are no events, use the messages tab as the default
           get().appActions.setSampleTab(kSampleMessagesTabId);
         }
       },
-      clearSelectedSample: () =>
+      getSelectedSample: () => {
+        const state = get().sample;
+        // Return from state if stored there, otherwise from ref
+        return state.sampleInState
+          ? state.selectedSampleObject
+          : selectedSampleRef.current;
+      },
+      clearSelectedSample: () => {
+        // Clear both the ref and the state
+        selectedSampleRef.current = undefined;
         set((state) => {
-          state.sample.selectedSample = undefined;
-        }),
+          state.sample.sample_identifier = undefined;
+          state.sample.selectedSampleObject = undefined;
+          state.sample.sampleInState = false;
+        });
+      },
       setSampleStatus: (status: SampleStatus) =>
         set((state) => {
           state.sample.sampleStatus = status;
@@ -73,10 +130,59 @@ export const createSampleSlice = (
         set((state) => {
           state.sample.sampleError = error;
         }),
+      setCollapsedEvents: (collapsed: Record<string, true>) => {
+        set((state) => {
+          state.sample.collapsedEvents = collapsed;
+        });
+      },
+      clearCollapsedEvents: () => {
+        set((state) => {
+          state.sample.collapsedEvents = null;
+        });
+      },
+      collapseEvent: (id: string, collapsed: boolean) => {
+        set((state) => {
+          if (state.sample.collapsedEvents === null) {
+            state.sample.collapsedEvents = {};
+          }
+          if (collapsed) {
+            state.sample.collapsedEvents[id] = true;
+          } else {
+            delete state.sample.collapsedEvents[id];
+          }
+        });
+      },
+      setCollapsedIds: (key: string, collapsed: Record<string, true>) => {
+        set((state) => {
+          state.sample.collapsedIdBuckets[key] = collapsed;
+        });
+      },
+      collapseId: (key: string, id: string, collapsed: boolean) => {
+        set((state) => {
+          if (state.sample.collapsedIdBuckets[key] === undefined) {
+            state.sample.collapsedIdBuckets[key] = {};
+          }
+          if (collapsed) {
+            state.sample.collapsedIdBuckets[key][id] = true;
+          } else {
+            delete state.sample.collapsedIdBuckets[key][id];
+          }
+        });
+      },
+      clearCollapsedIds: (key: string) => {
+        set((state) => {
+          delete state.sample.collapsedIdBuckets[key];
+        });
+      },
+
       pollSample: async (logFile: string, sampleSummary: SampleSummary) => {
         // Poll running sample
         const state = get();
-        if (state.log.loadedLog && state.sample.selectedSample) {
+        const sampleExists = state.sample.sampleInState
+          ? !!state.sample.selectedSampleObject
+          : !!selectedSampleRef.current;
+
+        if (state.log.loadedLog && sampleExists) {
           samplePolling.startPolling(logFile, sampleSummary);
         }
       },
@@ -85,6 +191,8 @@ export const createSampleSlice = (
 
         sampleActions.setSampleError(undefined);
         sampleActions.setSampleStatus("loading");
+        const state = get();
+
         try {
           if (sampleSummary.completed !== false) {
             log.debug(
@@ -95,8 +203,18 @@ export const createSampleSlice = (
               sampleSummary.id,
               sampleSummary.epoch,
             );
+            log.debug(
+              `LOADED COMPLETED SAMPLE: ${sampleSummary.id}-${sampleSummary.epoch}`,
+            );
             if (sample) {
               const migratedSample = resolveSample(sample);
+
+              if (
+                state.sample.sample_identifier?.id !== sample.id &&
+                state.sample.sample_identifier?.epoch !== sample.epoch
+              ) {
+                sampleActions.clearCollapsedEvents();
+              }
               sampleActions.setSelectedSample(migratedSample);
               sampleActions.setSampleStatus("ok");
             } else {
@@ -123,10 +241,19 @@ export const createSampleSlice = (
 
   const cleanup = () => {
     samplePolling.cleanup();
+    // Clear the ref when cleaning up
+    selectedSampleRef.current = undefined;
   };
   return [slice, cleanup];
 };
 
+export const handleRehydrate = (state: StoreState) => {
+  // Increment the reload counter if the sample is not in state
+  if (!state.sample.sampleInState) {
+    state.sample.sampleNeedsReload = state.sample.sampleNeedsReload + 1;
+  }
+};
+
 export const initializeSampleSlice = (
   set: (fn: (state: StoreState) => void) => void,
 ) => {

inspect_ai/_view/www/src/state/scoring.ts

@@ -37,7 +37,7 @@ const getScorersFromResults = (results?: EvalResults): ScorerInfo[] => {
 const getScorersFromSamples = (samples: SampleSummary[]): ScorerInfo[] => {
   // Find a sample with scores
   const scoredSample = samples.find((sample) => {
-    return !!sample.scores;
+    return !sample.error && sample.completed && !!sample.scores;
   });
 
   return Object.keys(scoredSample?.scores || {}).map((key) => ({

inspect_ai/_view/www/src/state/scrolling.ts

@@ -11,7 +11,7 @@ export function useStatefulScrollPosition<
 >(
   elementRef: RefObject<T | null>,
   elementKey: string,
-  delay = 500,
+  delay = 1000,
   scrollable = true,
 ) {
   const getScrollPosition = useStore(
@@ -62,12 +62,45 @@ export function useStatefulScrollPosition<
     const savedPosition = getScrollPosition(elementKey);
     if (savedPosition !== undefined) {
       log.debug(`Restoring scroll position`, savedPosition);
-      // Ensure the element has fully rendered
-      requestAnimationFrame(() => {
-        if (element.scrollTop !== savedPosition) {
-          element.scrollTop = savedPosition;
+
+      // Function to check and restore scroll position
+      const tryRestoreScroll = () => {
+        // Check if element has content to scroll (scrollHeight > clientHeight)
+        if (element.scrollHeight > element.clientHeight) {
+          if (element.scrollTop !== savedPosition) {
+            element.scrollTop = savedPosition;
+            log.debug(`Scroll position restored to ${savedPosition}`);
+          }
+          return true; // Successfully restored
         }
-      });
+        return false; // Not ready yet
+      };
+
+      // Try immediately once
+      if (!tryRestoreScroll()) {
+        // If not successful, set up polling with setTimeout for 1-second intervals
+        let attempts = 0;
+        const maxAttempts = 5; // Fewer attempts since we're waiting longer
+
+        const pollForRender = () => {
+          if (tryRestoreScroll() || attempts >= maxAttempts) {
+            // Either success or max attempts reached
+            if (attempts >= maxAttempts) {
+              log.debug(
+                `Failed to restore scroll after ${maxAttempts} attempts`,
+              );
+            }
+            return;
+          }
+
+          attempts++;
+          // Wait 1 second before trying again
+          setTimeout(pollForRender, 1000);
+        };
+
+        // Start polling after 1 second
+        setTimeout(pollForRender, 1000);
+      }
     }
 
     // Set up scroll listener

inspect_ai/_view/www/src/state/store.ts

@@ -1,3 +1,4 @@
+import { enableMapSet } from "immer";
 import { create, StoreApi, UseBoundStore } from "zustand";
 import { devtools, persist } from "zustand/middleware";
 import { immer } from "zustand/middleware/immer";
@@ -9,6 +10,7 @@ import { createLogSlice, initalializeLogSlice, LogSlice } from "./logSlice";
 import { createLogsSlice, initializeLogsSlice, LogsSlice } from "./logsSlice";
 import {
   createSampleSlice,
+  handleRehydrate,
   initializeSampleSlice,
   SampleSlice,
 } from "./sampleSlice";
@@ -52,6 +54,8 @@ export const initializeStore = (
   capabilities: Capabilities,
   storage?: ClientStorage,
 ) => {
+  enableMapSet();
+
   // Create the storage implementation
   const storageImplementation = {
     getItem: <T>(name: string): T | null => {
@@ -151,6 +155,7 @@ export const initializeStore = (
           version: 1,
           onRehydrateStorage: (state: StoreState) => {
             return (hydrationState, error) => {
+              handleRehydrate(state);
               log.debug("REHYDRATING STATE");
               if (error) {
                 log.debug("ERROR", { error });

inspect_ai/_view/www/src/state/store_filter.ts

@@ -1,3 +1,5 @@
+import { EvalSample } from "../@types/log";
+import { estimateSize } from "../utils/json";
 import { PersistedState } from "./store";
 
 export function filterState(state: PersistedState) {
@@ -6,31 +8,60 @@ export function filterState(state: PersistedState) {
   }
 
   // When saving state, we can't store vast amounts of data (like a large sample)
-  const filters = [filterLargeSample, filterLargeLogSummary];
+  const filters = [filterLargeLogSummary];
   return filters.reduce(
     (filteredState, filter) => filter(filteredState),
     state,
   );
 }
 
-// Filters the selected Sample if it is large
-function filterLargeSample(state: PersistedState): PersistedState {
-  if (!state || !state.sample || !state.sample.selectedSample) {
-    return state;
+export function isLargeSample(sample: EvalSample): boolean {
+  const storeKeys = countKeys(sample.store);
+  if (storeKeys > 5000) {
+    return true;
   }
 
-  const estimatedTotalSize = estimateSize(state.sample.selectedSample.messages);
-  if (estimatedTotalSize > 250000) {
-    return {
-      ...state,
-      sample: {
-        ...state.sample,
-        selectedSample: undefined,
-      },
-    };
-  } else {
-    return state;
+  const estimatedMessageSize = estimateSize(sample.messages);
+  if (estimatedMessageSize > 250000) {
+    return true;
   }
+
+  return true;
+}
+
+function countKeys(obj: unknown, options = { countArrayIndices: false }) {
+  // Base case: not an object or null
+  if (obj === null || typeof obj !== "object") {
+    return 0;
+  }
+
+  // Handle arrays
+  if (Array.isArray(obj)) {
+    let count = 0;
+    // Count array indices as keys if option is set
+    if (options.countArrayIndices) {
+      count += obj.length;
+    }
+    // Count keys in array elements that are objects
+    for (const item of obj) {
+      count += countKeys(item, options);
+    }
+    return count;
+  }
+
+  // For regular objects, count all own properties
+  let count = Object.keys(obj).length;
+
+  // Recursively count keys in nested objects
+  for (const key in obj) {
+    // Use type assertion to tell TypeScript that the key is valid
+    if (Object.prototype.hasOwnProperty.call(obj, key)) {
+      // Use type assertion (obj as Record<string, unknown>)
+      count += countKeys((obj as Record<string, unknown>)[key], options);
+    }
+  }
+
+  return count;
 }
 
 // Filters the selectedlog if it is too large
@@ -54,31 +85,3 @@ function filterLargeLogSummary(state: PersistedState): PersistedState {
     return state;
   }
 }
-
-function estimateSize(list: unknown[], frequency = 0.2) {
-  if (!list || list.length === 0) {
-    return 0;
-  }
-
-  // Total number of samples
-  const sampleSize = Math.ceil(list.length * frequency);
-
-  // Get a proper random sample without duplicates
-  const messageIndices = new Set<number>();
-  while (
-    messageIndices.size < sampleSize &&
-    messageIndices.size < list.length
-  ) {
-    const randomIndex = Math.floor(Math.random() * list.length);
-    messageIndices.add(randomIndex);
-  }
-
-  // Calculate size from sampled messages
-  const totalSize = Array.from(messageIndices).reduce((size, index) => {
-    return size + JSON.stringify(list[index]).length;
-  }, 0);
-
-  // Estimate total size based on sample
-  const estimatedTotalSize = (totalSize / sampleSize) * list.length;
-  return estimatedTotalSize;
-}

inspect_ai/_view/www/src/utils/debugging.ts

@@ -26,3 +26,98 @@ export function printCircularReferences(obj: Record<string, unknown>): void {
 
   detect(obj, "root");
 }
+
+export function findDifferences(
+  obj1: unknown,
+  obj2: unknown,
+  path = "",
+): string[] {
+  // Helper to build a readable path string
+  const makePath = (parent: string, key: string | number, isIndex = false) =>
+    parent
+      ? isIndex
+        ? `${parent}[${key}]`
+        : `${parent}.${key}`
+      : isIndex
+        ? `[${key}]`
+        : `${key}`;
+
+  // Primitive / simple equality check (Object.is handles NaN)
+  if (Object.is(obj1, obj2)) return [];
+
+  // Primitives or null → direct difference
+  if (
+    obj1 === null ||
+    obj2 === null ||
+    typeof obj1 !== "object" ||
+    typeof obj2 !== "object"
+  ) {
+    return [
+      `${path || "<root>"}: ${JSON.stringify(obj1)} → ${JSON.stringify(obj2)}`,
+    ];
+  }
+
+  // --- Arrays --------------------------------------------------------------
+  const isArr1 = Array.isArray(obj1);
+  const isArr2 = Array.isArray(obj2);
+  if (isArr1 || isArr2) {
+    if (isArr1 !== isArr2) {
+      return [`${path || "<root>"}: one is an array, the other is not`];
+    }
+
+    const diff: string[] = [];
+    const maxLen = Math.max(
+      (obj1 as unknown[]).length,
+      (obj2 as unknown[]).length,
+    );
+
+    if ((obj1 as unknown[]).length !== (obj2 as unknown[]).length) {
+      diff.push(
+        `${path || "<root>"}: array length ${
+          (obj1 as unknown[]).length
+        } vs ${(obj2 as unknown[]).length}`,
+      );
+    }
+
+    for (let i = 0; i < maxLen; i++) {
+      diff.push(
+        ...findDifferences(
+          (obj1 as unknown[])[i],
+          (obj2 as unknown[])[i],
+          makePath(path, i, true),
+        ),
+      );
+    }
+    return diff;
+  }
+
+  // --- Plain objects -------------------------------------------------------
+  const allKeys = new Set([
+    ...Object.keys(obj1 as Record<string, unknown>),
+    ...Object.keys(obj2 as Record<string, unknown>),
+  ]);
+
+  const diff: string[] = [];
+
+  for (const key of allKeys) {
+    const has1 = Object.prototype.hasOwnProperty.call(obj1, key);
+    const has2 = Object.prototype.hasOwnProperty.call(obj2, key);
+    const newPath = makePath(path, key);
+
+    if (!has1) {
+      diff.push(`${newPath}: property missing in first object`);
+    } else if (!has2) {
+      diff.push(`${newPath}: property missing in second object`);
+    } else {
+      diff.push(
+        ...findDifferences(
+          (obj1 as Record<string, unknown>)[key],
+          (obj2 as Record<string, unknown>)[key],
+          newPath,
+        ),
+      );
+    }
+  }
+
+  return diff;
+}

inspect_ai/_view/www/src/utils/format.ts

@@ -84,13 +84,13 @@ export const formatTime = (seconds: number): string => {
     const hours = Math.floor(seconds / (60 * 60));
     const minutes = Math.floor((seconds % (60 * 60)) / 60);
     const remainingSeconds = seconds % 60;
-    return `${hours} hr ${minutes} min ${remainingSeconds} sec`;
+    return `${hours} hr ${minutes} min ${Math.floor(remainingSeconds)} sec`;
   } else {
     const days = Math.floor(seconds / (60 * 60 * 24));
     const hours = Math.floor((seconds % (60 * 60 * 24)) / (60 * 60));
     const minutes = Math.floor((seconds % (60 * 60)) / 60);
     const remainingSeconds = seconds % 60;
-    return `${days} days ${hours} hr ${minutes} min ${remainingSeconds} sec`;
+    return `${days} days ${hours} hr ${minutes} min ${Math.floor(remainingSeconds)} sec`;
   }
 };
 

inspect_ai/_view/www/src/utils/json.ts

@@ -22,3 +22,32 @@ export const parsedJson = (text: string): unknown | undefined => {
   }
   return undefined;
 };
+
+// Estimates the size of a list of objects by sampling a subset of the list.
+export function estimateSize(list: unknown[], frequency = 0.2) {
+  if (!list || list.length === 0) {
+    return 0;
+  }
+
+  // Total number of samples
+  const sampleSize = Math.ceil(list.length * frequency);
+
+  // Get a proper random sample without duplicates
+  const messageIndices = new Set<number>();
+  while (
+    messageIndices.size < sampleSize &&
+    messageIndices.size < list.length
+  ) {
+    const randomIndex = Math.floor(Math.random() * list.length);
+    messageIndices.add(randomIndex);
+  }
+
+  // Calculate size from sampled messages
+  const totalSize = Array.from(messageIndices).reduce((size, index) => {
+    return size + JSON.stringify(list[index]).length;
+  }, 0);
+
+  // Estimate total size based on sample
+  const estimatedTotalSize = (totalSize / sampleSize) * list.length;
+  return estimatedTotalSize;
+}

inspect_ai/agent/__init__.py

@@ -1,4 +1,4 @@
-from ._agent import Agent, AgentState, agent, agent_with
+from ._agent import Agent, AgentState, agent, agent_with, is_agent
 from ._as_solver import as_solver
 from ._as_tool import as_tool
 from ._bridge.bridge import bridge
@@ -29,6 +29,7 @@ __all__ = [
     "AgentState",
     "agent",
     "agent_with",
+    "is_agent",
     "AgentPrompt",
     "AgentAttempts",
     "AgentContinue",

inspect_ai/agent/_agent.py

@@ -270,6 +270,18 @@ def agent_register(agent: Callable[P, Agent], name: str) -> Callable[P, Agent]:
 
 
 def is_agent(obj: Any) -> TypeGuard[Agent]:
+    """Check if an object is an Agent.
+
+    Determines if the provided object is registered as an Agent in the system registry.
+    When this function returns True, type checkers will recognize 'obj' as an Agent type.
+
+    Args:
+        obj: Object to check against the registry.
+
+    Returns:
+        True if the object is a registered Agent, False otherwise.
+        Acts as a TypeGuard to provide type narrowing for static type checkers.
+    """
     return is_registry_object(obj, type="agent")