inspect-ai 0.3.81__py3-none-any.whl → 0.3.82__py3-none-any.whl

inspect_ai/_view/www/src/state/store.ts

@@ -0,0 +1,168 @@
+import { create, StoreApi, UseBoundStore } from "zustand";
+import { devtools, persist } from "zustand/middleware";
+import { immer } from "zustand/middleware/immer";
+import { Capabilities, ClientAPI, ClientStorage } from "../api/types";
+import { createLogger } from "../utils/logger";
+import { AppSlice, createAppSlice, initializeAppSlice } from "./appSlice";
+import { createLogSlice, initalializeLogSlice, LogSlice } from "./logSlice";
+import { createLogsSlice, initializeLogsSlice, LogsSlice } from "./logsSlice";
+import {
+  createSampleSlice,
+  initializeSampleSlice,
+  SampleSlice,
+} from "./sampleSlice";
+import { filterState } from "./store_filter";
+
+const log = createLogger("store");
+
+export interface StoreState extends AppSlice, LogsSlice, LogSlice, SampleSlice {
+  // The shared api
+  api?: ClientAPI | null;
+
+  // Global actions
+  initialize: (api: ClientAPI, capabilities: Capabilities) => void;
+  cleanup: () => void;
+}
+
+// The data that will actually be persisted
+export type PersistedState = {
+  app: AppSlice["app"];
+  log: LogSlice["log"];
+  logs: LogsSlice["logs"];
+  sample: SampleSlice["sample"];
+};
+
+// The store implementation (this will be set when the store is initialized)
+let storeImplementation: UseBoundStore<StoreApi<StoreState>> | null = null;
+
+// Create a proxy store that forwards calls to the real store once initialized
+export const useStore = ((selector?: any) => {
+  if (!storeImplementation) {
+    throw new Error(
+      "Store accessed before initialization. Call initializeStore first.",
+    );
+  }
+  return selector ? storeImplementation(selector) : storeImplementation();
+}) as UseBoundStore<StoreApi<StoreState>>;
+
+// Initialize the store
+export const initializeStore = (
+  api: ClientAPI,
+  capabilities: Capabilities,
+  storage?: ClientStorage,
+) => {
+  // Create the storage implementation
+  const storageImplementation = {
+    getItem: <T>(name: string): T | null => {
+      return storage ? (storage.getItem(name) as T) : null;
+    },
+    setItem: <T>(name: string, value: T): void => {
+      if (storage) {
+        storage.setItem(name, value);
+      }
+    },
+    removeItem: (name: string): void => {
+      if (storage) {
+        storage.removeItem(name);
+      }
+    },
+  };
+
+  // Create the actual store
+  const store = create<StoreState>()(
+    devtools(
+      persist(
+        immer((set, get, store) => {
+          const [appSlice, appCleanup] = createAppSlice(
+            set as (fn: (state: StoreState) => void) => void,
+            get,
+            store,
+          );
+          const [logsSlice, logsCleanup] = createLogsSlice(
+            set as (fn: (state: StoreState) => void) => void,
+            get,
+            store,
+          );
+          const [logSlice, logCleanup] = createLogSlice(
+            set as (fn: (state: StoreState) => void) => void,
+            get,
+            store,
+          );
+          const [sampleSlice, sampleCleanup] = createSampleSlice(
+            set as (fn: (state: StoreState) => void) => void,
+            get,
+            store,
+          );
+
+          return {
+            // Shared state
+            api: null,
+
+            // Initialize
+            initialize: (api, capabilities) => {
+              set((state) => {
+                state.api = api;
+              });
+
+              // Initialize application slices
+              initializeAppSlice(
+                set as (fn: (state: StoreState) => void) => void,
+                capabilities,
+              );
+              initializeLogsSlice(
+                set as (fn: (state: StoreState) => void) => void,
+              );
+              initalializeLogSlice(
+                set as (fn: (state: StoreState) => void) => void,
+              );
+              initializeSampleSlice(
+                set as (fn: (state: StoreState) => void) => void,
+              );
+            },
+
+            // Create the slices and merge them in
+            ...appSlice,
+            ...logsSlice,
+            ...logSlice,
+            ...sampleSlice,
+
+            cleanup: () => {
+              appCleanup();
+              logsCleanup();
+              logCleanup();
+              sampleCleanup();
+            },
+          };
+        }),
+        {
+          name: "app-storage",
+          storage: storageImplementation,
+          partialize: (state) => {
+            const persisted: PersistedState = filterState({
+              app: state.app,
+              log: state.log,
+              logs: state.logs,
+              sample: state.sample,
+            });
+            return persisted as unknown as StoreState;
+          },
+          version: 1,
+          onRehydrateStorage: (state: StoreState) => {
+            return (hydrationState, error) => {
+              log.debug("REHYDRATING STATE");
+              if (error) {
+                log.debug("ERROR", { error });
+              } else {
+                log.debug("STATE", { state, hydrationState });
+              }
+            };
+          },
+        },
+      ),
+    ),
+  );
+
+  // Set the implementation and initialize it
+  storeImplementation = store as UseBoundStore<StoreApi<StoreState>>;
+  store.getState().initialize(api, capabilities);
+};

inspect_ai/_view/www/src/state/store_filter.ts

@@ -0,0 +1,84 @@
+import { PersistedState } from "./store";
+
+export function filterState(state: PersistedState) {
+  if (!state) {
+    return state;
+  }
+
+  // When saving state, we can't store vast amounts of data (like a large sample)
+  const filters = [filterLargeSample, filterLargeLogSummary];
+  return filters.reduce(
+    (filteredState, filter) => filter(filteredState),
+    state,
+  );
+}
+
+// Filters the selected Sample if it is large
+function filterLargeSample(state: PersistedState): PersistedState {
+  if (!state || !state.sample || !state.sample.selectedSample) {
+    return state;
+  }
+
+  const estimatedTotalSize = estimateSize(state.sample.selectedSample.messages);
+  if (estimatedTotalSize > 250000) {
+    return {
+      ...state,
+      sample: {
+        ...state.sample,
+        selectedSample: undefined,
+      },
+    };
+  } else {
+    return state;
+  }
+}
+
+// Filters the selectedlog if it is too large
+function filterLargeLogSummary(state: PersistedState): PersistedState {
+  if (!state || !state.log || !state.log.selectedLogSummary) {
+    return state;
+  }
+
+  const estimatedSize = estimateSize(
+    state.log.selectedLogSummary.sampleSummaries,
+  );
+  if (estimatedSize > 250000) {
+    return {
+      ...state,
+      log: {
+        ...state.log,
+        selectedLogSummary: undefined,
+      },
+    };
+  } else {
+    return state;
+  }
+}
+
+function estimateSize(list: unknown[], frequency = 0.2) {
+  if (!list || list.length === 0) {
+    return 0;
+  }
+
+  // Total number of samples
+  const sampleSize = Math.ceil(list.length * frequency);
+
+  // Get a proper random sample without duplicates
+  const messageIndices = new Set<number>();
+  while (
+    messageIndices.size < sampleSize &&
+    messageIndices.size < list.length
+  ) {
+    const randomIndex = Math.floor(Math.random() * list.length);
+    messageIndices.add(randomIndex);
+  }
+
+  // Calculate size from sampled messages
+  const totalSize = Array.from(messageIndices).reduce((size, index) => {
+    return size + JSON.stringify(list[index]).length;
+  }, 0);
+
+  // Estimate total size based on sample
+  const estimatedTotalSize = (totalSize / sampleSize) * list.length;
+  return estimatedTotalSize;
+}

inspect_ai/_view/www/src/state/utils.ts

@@ -0,0 +1,23 @@
+import { SampleSummary } from "../api/types";
+
+// Function to merge log samples with pending samples
+export const mergeSampleSummaries = (
+  logSamples: SampleSummary[],
+  pendingSamples: SampleSummary[],
+) => {
+  // Create a map of existing sample IDs to avoid duplicates
+  const existingSampleIds = new Set(
+    logSamples.map((sample) => `${sample.id}-${sample.epoch}`),
+  );
+
+  // Filter out any pending samples that already exist in the log
+  const uniquePendingSamples = pendingSamples
+    .filter((sample) => !existingSampleIds.has(`${sample.id}-${sample.epoch}`))
+    .map((sample) => {
+      // Always mark pending items as incomplete to be sure we trigger polling
+      return { ...sample, completed: false };
+    });
+
+  // Combine and return all samples
+  return [...logSamples, ...uniquePendingSamples];
+};

inspect_ai/_view/www/src/storage/index.ts

@@ -0,0 +1,26 @@
+import { ClientStorage } from "../api/types";
+import { PersistedState } from "../state/store";
+import { getVscodeApi } from "../utils/vscode";
+
+const resolveStorage = (): ClientStorage | undefined => {
+  const vscodeApi = getVscodeApi();
+  if (vscodeApi) {
+    return {
+      getItem: (_name: string) => {
+        const state = vscodeApi.getState() as PersistedState;
+        return state;
+      },
+      setItem: (_name: string, value: unknown) => {
+        // TODO: This is pretty gnarly type hijinks
+        const valObj = value as { state: PersistedState; version: number };
+        vscodeApi.setState(valObj);
+      },
+      removeItem: (_name: string) => {
+        vscodeApi.setState(null);
+      },
+    };
+  }
+  return undefined;
+};
+
+export default resolveStorage();

inspect_ai/_view/www/src/types/log.d.ts

@@ -44,6 +44,7 @@ export type SandboxCleanup = boolean | null;
 export type LogSamples = boolean | null;
 export type LogImages = boolean | null;
 export type LogBuffer = number | null;
+export type LogShared = number | null;
 export type ScoreDisplay = boolean | null;
 export type Type1 = "git";
 export type Origin = string;
@@ -627,6 +628,7 @@ export interface EvalConfig {
   log_samples: LogSamples;
   log_images: LogImages;
   log_buffer: LogBuffer;
+  log_shared: LogShared;
   score_display: ScoreDisplay;
 }
 export interface ApprovalPolicyConfig {

inspect_ai/_view/www/src/types.ts

@@ -1,49 +1,105 @@
+import { StateSnapshot } from "react-virtuoso";
 import {
+  AttachmentData,
   EvalLogHeader,
   EvalSummary,
+  EventData,
   LogFiles,
+  PendingSamples,
   SampleSummary,
 } from "./api/types";
-import { ContentImage, ContentText, EvalSample } from "./types/log";
-
-export interface ApplicationState {
-  logs?: LogFiles;
-  selectedLogIndex?: number;
-  logHeaders?: Record<string, EvalLogHeader>;
-  headersLoading?: boolean;
-  selectedLog?: CurrentLog;
-  selectedWorkspaceTab?: string;
-  selectedSampleIndex?: number;
-  selectedSample?: EvalSample;
-  sampleStatus?: "loading" | "ok" | "error";
-  sampleError?: Error;
-  selectedSampleTab?: string;
-  sampleScrollPosition?: number;
-  showingSampleDialog?: boolean;
-  status?: AppStatus;
-  offcanvas?: boolean;
-  showFind?: boolean;
-  filter?: ScoreFilter;
-  epoch?: string;
-  sort?: string;
-  scores?: ScoreLabel[];
+import { ScorerInfo } from "./scoring/utils";
+import {
+  ApprovalEvent,
+  ContentImage,
+  ContentText,
+  EvalSample,
+  InfoEvent,
+  LoggerEvent,
+  ModelEvent,
+  SampleInitEvent,
+  SampleLimitEvent,
+  SandboxEvent,
+  ScoreEvent,
+  StateEvent,
+  StepEvent,
+  StoreEvent,
+  SubtaskEvent,
+  ToolEvent,
+} from "./types/log";
+
+export interface AppState {
+  status: AppStatus;
+  offcanvas: boolean;
+  showFind: boolean;
+  tabs: {
+    workspace: string;
+    sample: string;
+  };
+  dialogs: {
+    sample: boolean;
+  };
+  scrollPositions: Record<string, number>;
+  listPositions: Record<string, StateSnapshot>;
+  collapsed: Record<string, boolean>;
+  messages: Record<string, boolean>;
+  propertyBags: Record<string, Record<string, unknown>>;
+}
+
+export interface LogsState {
+  logs: LogFiles;
+  logHeaders: Record<string, EvalLogHeader>;
+  headersLoading: boolean;
+  selectedLogIndex: number;
+}
+
+export interface LogState {
+  loadedLog?: string;
+
+  selectedSampleIndex: number;
+  selectedLogSummary?: EvalSummary;
+  pendingSampleSummaries?: PendingSamples;
+
+  filter: ScoreFilter;
+  epoch: string;
+  sort: string;
   score?: ScoreLabel;
-  filteredSamples?: SampleSummary[];
-  groupBy?: "none" | "epoch" | "sample";
-  groupByOrder?: "asc" | "desc";
-  workspaceTabScrollPosition?: Record<string, number>;
+  scores?: ScorerInfo[];
 }
 
+export type SampleStatus = "ok" | "loading" | "streaming" | "error";
+
+export interface SampleState {
+  selectedSample: EvalSample | undefined;
+  sampleStatus: SampleStatus;
+  sampleError: Error | undefined;
+
+  // Events and attachments
+  runningEvents: Event[];
+}
+
+export type Event =
+  | SampleInitEvent
+  | SampleLimitEvent
+  | SandboxEvent
+  | StateEvent
+  | StoreEvent
+  | ModelEvent
+  | ToolEvent
+  | ApprovalEvent
+  | InputEvent
+  | ScoreEvent
+  | ErrorEvent
+  | LoggerEvent
+  | InfoEvent
+  | StepEvent
+  | SubtaskEvent;
+
 export interface AppStatus {
   loading: boolean;
   error?: Error;
 }
 
-export interface Capabilities {
-  downloadFiles: boolean;
-  webWorkers: boolean;
-}
-
 export interface CurrentLog {
   name: string;
   contents: EvalSummary;
@@ -69,3 +125,9 @@ export interface ContentTool {
   type: "tool";
   content: (ContentImage | ContentText)[];
 }
+
+export interface RunningSampleData {
+  events: Map<string, EventData>;
+  attachments: Map<string, AttachmentData>;
+  summary?: SampleSummary;
+}

inspect_ai/_view/www/src/utils/attachments.ts

@@ -1,40 +1,75 @@
-/**
- * Resolves individual value by replacing protocol references with attachment content
- */
-export const resolveAttachments = (
-  value: any,
+export const resolveAttachments = <T>(
+  value: T,
   attachments: Record<string, string>,
-): any => {
-  const kContentProtocol = "tc://";
-  const kAttachmentProtocol = "attachment://";
+): T => {
+  const CONTENT_PROTOCOL = "tc://";
+  const ATTACHMENT_PROTOCOL = "attachment://";
+
+  // Handle null or undefined early
+  if (value === null || value === undefined) {
+    return value;
+  }
 
   // Handle arrays recursively
   if (Array.isArray(value)) {
-    return value.map((v) => resolveAttachments(v, attachments));
+    let hasChanged = false;
+    const resolvedArray = value.map((v) => {
+      const resolved = resolveAttachments(v, attachments);
+      if (resolved !== v) hasChanged = true;
+      return resolved;
+    });
+
+    // Only return the new array if something actually changed
+    return hasChanged ? (resolvedArray as unknown as T) : value;
   }
 
-  // Handle objects recursively
-  if (value && typeof value === "object") {
+  // Handle objects recursively, but skip Date instances and other special object types
+  if (
+    typeof value === "object" &&
+    !(value instanceof Date) &&
+    !(value instanceof RegExp)
+  ) {
+    let hasChanged = false;
     const resolvedObject: Record<string, unknown> = {};
-    for (const key of Object.keys(value)) {
-      resolvedObject[key] = resolveAttachments(value[key], attachments);
+
+    for (const [key, val] of Object.entries(value)) {
+      const resolved = resolveAttachments(val, attachments);
+      resolvedObject[key] = resolved;
+
+      // Track if anything changed
+      if (resolved !== val) hasChanged = true;
     }
-    return resolvedObject;
+
+    // Only return the new object if something actually changed
+    return hasChanged ? (resolvedObject as unknown as T) : value;
   }
 
   // Handle string values with protocol references
   if (typeof value === "string") {
-    let resolvedValue = value;
-    if (resolvedValue.startsWith(kContentProtocol)) {
-      resolvedValue = resolvedValue.replace(
-        kContentProtocol,
-        kAttachmentProtocol,
-      );
+    // Check if the string starts with the content protocol
+    if (value.startsWith(CONTENT_PROTOCOL)) {
+      const updatedValue = value.replace(CONTENT_PROTOCOL, ATTACHMENT_PROTOCOL);
+
+      // Now check if it's an attachment reference
+      if (updatedValue.startsWith(ATTACHMENT_PROTOCOL)) {
+        const attachmentId = updatedValue.slice(ATTACHMENT_PROTOCOL.length);
+        const attachment = attachments[attachmentId];
+
+        // Return the attachment content if it exists, otherwise return the original string
+        return (attachment !== undefined ? attachment : value) as unknown as T;
+      }
+
+      return updatedValue as unknown as T;
     }
-    if (resolvedValue.startsWith(kAttachmentProtocol)) {
-      return attachments[resolvedValue.replace(kAttachmentProtocol, "")];
+
+    // Check if it's directly an attachment reference
+    if (value.startsWith(ATTACHMENT_PROTOCOL)) {
+      const attachmentId = value.slice(ATTACHMENT_PROTOCOL.length);
+      const attachment = attachments[attachmentId];
+
+      // Return the attachment content if it exists, otherwise return the original string
+      return (attachment !== undefined ? attachment : value) as unknown as T;
     }
-    return resolvedValue;
   }
 
   // Return unchanged for other types

inspect_ai/_view/www/src/utils/logger.ts

@@ -0,0 +1,52 @@
+// This will be replaced at build time with a boolean value
+declare const __DEV_WATCH__: boolean;
+declare const __LOGGING_FILTER__: string;
+
+const getEnabledNamespaces = () => {
+  // Split by comma and filter out empty strings
+  return __LOGGING_FILTER__
+    .split(",")
+    .map((ns) => ns.trim())
+    .filter(Boolean);
+};
+
+const ENABLED_NAMESPACES = new Set<string>(getEnabledNamespaces());
+const filterNameSpace = (namespace: string) => {
+  if (ENABLED_NAMESPACES.has("*")) return true;
+
+  return ENABLED_NAMESPACES.has(namespace);
+};
+
+// Create a logger for a specific namespace
+export const createLogger = (namespace: string) => {
+  // Logger functions that only activate in dev-watch mode
+  const logger = {
+    debug: (message: string, ...args: any[]) => {
+      if (__DEV_WATCH__ && filterNameSpace(namespace))
+        console.debug(`[${namespace}] ${message}`, ...args);
+    },
+
+    info: (message: string, ...args: any[]) => {
+      if (__DEV_WATCH__ && filterNameSpace(namespace))
+        console.info(`[${namespace}] ${message}`, ...args);
+    },
+
+    warn: (message: string, ...args: any[]) => {
+      if (__DEV_WATCH__ && filterNameSpace(namespace))
+        console.warn(`[${namespace}] ${message}`, ...args);
+    },
+
+    // Always log errors, even in production
+    error: (message: string, ...args: any[]) => {
+      console.error(`[${namespace}] ${message}`, ...args);
+    },
+
+    // Lazy evaluation for expensive logs
+    debugIf: (fn: () => string) => {
+      if (__DEV_WATCH__ && filterNameSpace(namespace))
+        console.debug(`[${namespace}] ${fn()}`);
+    },
+  };
+
+  return logger;
+};