inspect-ai 0.3.81__py3-none-any.whl → 0.3.82__py3-none-any.whl

inspect_ai/_view/www/src/state/logPolling.ts

@@ -0,0 +1,196 @@
+import { createLogger } from "../utils/logger";
+import { createPolling } from "../utils/polling";
+import { StoreState } from "./store";
+
+// The logger
+const log = createLogger("logPolling");
+
+const kRetries = 10;
+const kPollingInterval = 2;
+
+export function createLogPolling(
+  get: () => StoreState,
+  set: (fn: (state: StoreState) => void) => void,
+) {
+  // Tracks the currently polling instance
+  let currentPolling: ReturnType<typeof createPolling> | null = null;
+
+  // handle aborts
+  let abortController: AbortController;
+
+  // Refresh logs and clear pending in a single transaction
+  const refreshLog = async (logFileName: string, clearPending = false) => {
+    if (abortController?.signal.aborted) {
+      return false;
+    }
+
+    const state = get();
+    const api = state.api;
+    const selectedLogFile = state.logsActions.getSelectedLogFile();
+
+    if (!api || !selectedLogFile) {
+      return false;
+    }
+
+    log.debug(`refresh: ${selectedLogFile}`);
+
+    try {
+      const logContents = await api.get_log_summary(selectedLogFile);
+
+      set((state) => {
+        // Set the log summary
+        state.log.selectedLogSummary = logContents;
+        log.debug(
+          `Setting refreshed summary ${logContents.sampleSummaries.length} samples`,
+          logContents,
+        );
+
+        // Clear pending summaries if requested
+        if (clearPending) {
+          const pendingSampleSummaries = state.log.pendingSampleSummaries;
+          if ((pendingSampleSummaries?.samples.length || 0) > 0) {
+            log.debug(
+              `Clearing pending summaries during refresh for: ${logFileName}`,
+            );
+            state.log.pendingSampleSummaries = {
+              samples: [],
+              refresh: pendingSampleSummaries?.refresh || 2,
+            };
+          }
+        }
+      });
+
+      return true;
+    } catch (error) {
+      log.error("Error refreshing log:", error);
+      return false;
+    }
+  };
+
+  // Function to start polling for a specific log file
+  const startPolling = (logFileName: string) => {
+    // Stop any existing polling
+    if (currentPolling) {
+      currentPolling.stop();
+    }
+
+    // note that we're active
+    abortController = new AbortController();
+
+    // Track whether we ever polled
+    let loadedPendingSamples = false;
+
+    // Create a new polling instance
+    currentPolling = createPolling(
+      `PendingSamples-${logFileName}`,
+      async () => {
+        if (abortController.signal.aborted) {
+          log.debug(`Component unmounted, stopping poll for: ${logFileName}`);
+          return false;
+        }
+
+        // The state for polling
+        const state = get();
+
+        // Don't proceed if API doesn't support it
+        const api = state.api;
+        if (!api?.get_log_pending_samples) {
+          return false;
+        }
+
+        if (abortController.signal.aborted) {
+          return false;
+        }
+
+        // Fetch pending samples
+        log.debug(`Polling running samples: ${logFileName}`);
+        const currentEtag = get().log.pendingSampleSummaries?.etag;
+        const pendingSamples = await api.get_log_pending_samples(
+          logFileName,
+          currentEtag,
+        );
+        log.debug(`Received pending samples`, pendingSamples);
+
+        if (abortController.signal.aborted) {
+          return false;
+        }
+
+        if (pendingSamples.status === "OK" && pendingSamples.pendingSamples) {
+          loadedPendingSamples = true;
+
+          // Update state with new pending samples
+          set((state) => {
+            state.log.pendingSampleSummaries = pendingSamples.pendingSamples;
+          });
+
+          // Refresh the log to get latest data
+          await refreshLog(logFileName, false);
+
+          // Continue polling
+          return true;
+        } else if (pendingSamples.status === "NotFound") {
+          // The eval has completed (no more events/pending samples will be delivered)
+          log.debug(`Stop polling running samples: ${logFileName}`);
+
+          // Clear pending summaries and refresh in one transaction
+          if (loadedPendingSamples) {
+            await refreshLog(logFileName, true);
+          }
+
+          // Stop polling
+          return false;
+        }
+
+        // Continue polling by default
+        return true;
+      },
+      {
+        maxRetries: kRetries,
+        interval: get().log.pendingSampleSummaries?.refresh || kPollingInterval,
+      },
+    );
+
+    // Start the polling
+    currentPolling.start();
+  };
+
+  // Clear pending summaries (now using the transactional approach)
+  const clearPendingSummaries = (logFileName: string) => {
+    if (abortController.signal.aborted) {
+      return false;
+    }
+
+    const pendingSampleSummaries = get().log.pendingSampleSummaries;
+    if ((pendingSampleSummaries?.samples.length || 0) > 0) {
+      log.debug(`Clear pending: ${logFileName}`);
+      return refreshLog(logFileName, true);
+    }
+
+    return false;
+  };
+
+  // Stop polling
+  const stopPolling = () => {
+    if (currentPolling) {
+      currentPolling.stop();
+      currentPolling = null;
+    }
+  };
+
+  // Method to call when component unmounts
+  const cleanup = () => {
+    log.debug(`Cleanup`);
+    abortController.abort();
+    stopPolling();
+  };
+
+  return {
+    startPolling,
+    stopPolling,
+    clearPendingSummaries,
+    cleanup,
+    // Expose the refresh function so components can use it directly
+    refreshLog: (clearPending = false) =>
+      refreshLog(get().logsActions.getSelectedLogFile() || "", clearPending),
+  };
+}

inspect_ai/_view/www/src/state/logSlice.ts

@@ -0,0 +1,214 @@
+import { EvalSummary, PendingSamples } from "../api/types";
+import { kDefaultSort, kInfoWorkspaceTabId } from "../constants";
+import { ScorerInfo } from "../scoring/utils";
+import { LogState, ScoreFilter, ScoreLabel } from "../types";
+import { createLogger } from "../utils/logger";
+import { createLogPolling } from "./logPolling";
+import { StoreState } from "./store";
+
+const log = createLogger("logSlice");
+
+export interface LogSlice {
+  log: LogState;
+  logActions: {
+    selectSample: (index: number) => void;
+
+    // Set the selected log summary
+    setSelectedLogSummary: (summary: EvalSummary) => void;
+
+    // Clear the selected log summary
+    clearSelectedLogSummary: () => void;
+
+    // Update pending sample information
+    setPendingSampleSummaries: (samples: PendingSamples) => void;
+
+    // Set filter criteria
+    setFilter: (filter: ScoreFilter) => void;
+
+    // Set epoch filter
+    setEpoch: (epoch: string) => void;
+
+    // Set sort order and update groupBy
+    setSort: (sort: string) => void;
+
+    // Set score label
+    setScore: (score: ScoreLabel) => void;
+
+    // Set available scores
+    setScores: (scores: ScorerInfo[]) => void;
+
+    // Reset filter state to defaults
+    resetFiltering: () => void;
+
+    // Load log
+    loadLog: (logFileName: string) => Promise<void>;
+
+    // Refresh the current log
+    refreshLog: () => Promise<void>;
+  };
+}
+
+// Initial state
+const initialState = {
+  // Log state
+  selectedSampleIndex: -1,
+  selectedLogSummary: undefined,
+  pendingSampleSummaries: undefined,
+  loadedLog: undefined,
+
+  // Filter state
+  filter: {},
+  epoch: "all",
+  sort: kDefaultSort,
+  score: undefined,
+  scores: undefined,
+};
+
+// Create the app slice using StoreState directly
+export const createLogSlice = (
+  set: (fn: (state: StoreState) => void) => void,
+  get: () => StoreState,
+  _store: any,
+): [LogSlice, () => void] => {
+  const logPolling = createLogPolling(get, set);
+
+  const slice = {
+    // State
+    log: initialState,
+
+    // Actions
+    logActions: {
+      selectSample: (index: number) =>
+        set((state) => {
+          state.log.selectedSampleIndex = index;
+        }),
+
+      setSelectedLogSummary: (selectedLogSummary: EvalSummary) => {
+        set((state) => {
+          state.log.selectedLogSummary = selectedLogSummary;
+        });
+
+        if (
+          selectedLogSummary.status !== "started" &&
+          selectedLogSummary.sampleSummaries.length === 0
+        ) {
+          // If there are no samples, use the workspace tab id by default
+          get().appActions.setWorkspaceTab(kInfoWorkspaceTabId);
+        }
+      },
+
+      clearSelectedLogSummary: () => {
+        set((state) => {
+          state.log.selectedLogSummary = undefined;
+        });
+      },
+      setPendingSampleSummaries: (pendingSampleSummaries: PendingSamples) =>
+        set((state) => {
+          state.log.pendingSampleSummaries = pendingSampleSummaries;
+        }),
+
+      setFilter: (filter: ScoreFilter) =>
+        set((state) => {
+          state.log.filter = filter;
+        }),
+      setEpoch: (epoch: string) =>
+        set((state) => {
+          state.log.epoch = epoch;
+        }),
+      setSort: (sort: string) =>
+        set((state) => {
+          state.log.sort = sort;
+        }),
+      setScore: (score: ScoreLabel) =>
+        set((state) => {
+          state.log.score = score;
+        }),
+      setScores: (scores: ScorerInfo[]) =>
+        set((state) => {
+          state.log.scores = scores;
+        }),
+      resetFiltering: () =>
+        set((state) => {
+          state.log.filter = {};
+          state.log.epoch = "all";
+          state.log.sort = kDefaultSort;
+          state.log.score = undefined;
+        }),
+
+      loadLog: async (logFileName: string) => {
+        const state = get();
+        const api = state.api;
+
+        if (!api) {
+          console.error("API not initialized in Store");
+          return;
+        }
+
+        log.debug(`Load log: ${logFileName}`);
+        try {
+          const logContents = await api.get_log_summary(logFileName);
+          state.logActions.setSelectedLogSummary(logContents);
+          state.logActions.setEpoch;
+
+          // Push the updated header information up
+          const header = {
+            [logFileName]: {
+              version: logContents.version,
+              status: logContents.status,
+              eval: logContents.eval,
+              plan: logContents.plan,
+              results:
+                logContents.results !== null ? logContents.results : undefined,
+              stats: logContents.stats,
+              error: logContents.error !== null ? logContents.error : undefined,
+            },
+          };
+
+          state.logsActions.updateLogHeaders(header);
+          set((state) => {
+            state.log.loadedLog = logFileName;
+          }),
+            // Start polling for pending samples
+            logPolling.startPolling(logFileName);
+        } catch (error) {
+          log.error("Error loading log:", error);
+        }
+      },
+
+      refreshLog: async () => {
+        const state = get();
+        const api = state.api;
+        const selectedLogFile = state.logsActions.getSelectedLogFile();
+
+        if (!api || !selectedLogFile) {
+          return;
+        }
+
+        log.debug(`refresh: ${selectedLogFile}`);
+        try {
+          const logContents = await api.get_log_summary(selectedLogFile);
+          state.logActions.setSelectedLogSummary(logContents);
+        } catch (error) {
+          log.error("Error refreshing log:", error);
+        }
+      },
+    },
+  } as const;
+
+  const cleanup = () => {
+    logPolling.cleanup();
+  };
+
+  return [slice, cleanup];
+};
+
+// Initialize app slice with StoreState
+export const initalializeLogSlice = (
+  set: (fn: (state: StoreState) => void) => void,
+) => {
+  set((state) => {
+    if (!state.log) {
+      state.log = initialState;
+    }
+  });
+};

inspect_ai/_view/www/src/state/logsPolling.ts

@@ -0,0 +1,118 @@
+import { EvalLogHeader, LogFiles } from "../api/types";
+import { createLogger } from "../utils/logger";
+import { createPolling } from "../utils/polling";
+import { StoreState } from "./store";
+
+// The logger
+const log = createLogger("logsPolling");
+
+export function createLogsPolling(
+  get: () => StoreState,
+  _set: (fn: (state: StoreState) => void) => void,
+) {
+  // Tracks the currently polling instance
+  let currentPolling: ReturnType<typeof createPolling> | null = null;
+
+  // Are we active
+  let isActive = true;
+
+  // Function to start polling for a specific log file
+  const startPolling = (logFiles: LogFiles) => {
+    // Get the api
+    const api = get().api;
+    if (!api) {
+      throw new Error("Failed to start polling - no API");
+    }
+
+    // Stop any existing polling
+    if (currentPolling) {
+      currentPolling.stop();
+    }
+
+    // note that we're active
+    isActive = true;
+
+    log.debug("LOADING HEADERS");
+    get().logsActions.setHeadersLoading(true);
+
+    // Group into chunks
+    const chunkSize = 8;
+    const fileLists: string[][] = [];
+
+    for (let i = 0; i < logFiles.files.length; i += chunkSize) {
+      const chunk = logFiles.files
+        .slice(i, i + chunkSize)
+        .map((logFile) => logFile.name);
+      fileLists.push(chunk);
+    }
+    const totalLen = fileLists.length;
+
+    // Create a new polling instance
+    currentPolling = createPolling(
+      `LogHeaders`,
+      async () => {
+        if (!isActive) {
+          get().logsActions.setHeadersLoading(false);
+          return false; // Stop polling
+        }
+
+        log.debug(`POLL HEADERS`);
+
+        const currentFileList = fileLists.shift();
+        if (currentFileList) {
+          log.debug(
+            `LOADING ${totalLen - fileLists.length} of ${totalLen} CHUNKS`,
+          );
+          const headers = await api.get_log_headers(currentFileList);
+          const updatedHeaders: Record<string, EvalLogHeader> = {};
+
+          headers.forEach((header, index) => {
+            const logFile = currentFileList[index];
+            updatedHeaders[logFile] = header as EvalLogHeader;
+          });
+          get().logsActions.updateLogHeaders(updatedHeaders);
+        } else {
+          // Stop polling
+          get().logsActions.setHeadersLoading(false);
+          return false;
+        }
+
+        if (!isActive) {
+          get().logsActions.setHeadersLoading(false);
+          return false; // Stop polling
+        }
+
+        // Continue polling by default
+        return true;
+      },
+      {
+        maxRetries: 10,
+        interval: 5,
+      },
+    );
+
+    // Start the polling
+    currentPolling.start();
+  };
+
+  // Stop polling
+  const stopPolling = () => {
+    if (currentPolling) {
+      currentPolling.stop();
+      currentPolling = null;
+    }
+  };
+
+  // Method to call when component unmounts
+  const cleanup = () => {
+    log.debug(`CLEANUP`);
+    isActive = false;
+    stopPolling();
+  };
+
+  return {
+    startPolling,
+    stopPolling,
+    cleanup,
+  };
+}

inspect_ai/_view/www/src/state/logsSlice.ts

@@ -0,0 +1,181 @@
+import { EvalLogHeader, LogFiles } from "../api/types";
+import { LogsState } from "../types";
+import { createLogger } from "../utils/logger";
+import { createLogsPolling } from "./logsPolling";
+import { StoreState } from "./store";
+
+const log = createLogger("Log Slice");
+
+export interface LogsSlice {
+  logs: LogsState;
+  logsActions: {
+    // Update State
+    setLogs: (logs: LogFiles) => void;
+    setLogHeaders: (headers: Record<string, EvalLogHeader>) => void;
+    setHeadersLoading: (loading: boolean) => void;
+    setSelectedLogIndex: (index: number) => void;
+    setSelectedLogFile: (logUrl: string) => void;
+    updateLogHeaders: (headers: Record<string, EvalLogHeader>) => void;
+
+    // Fetch or update logs
+    refreshLogs: () => Promise<void>;
+    selectLogFile: (logUrl: string) => Promise<void>;
+    loadLogs: () => Promise<LogFiles>;
+
+    // Computed values
+    getSelectedLogFile: () => string | undefined;
+  };
+}
+
+const initialState: LogsState = {
+  logs: { log_dir: "", files: [] },
+  logHeaders: {},
+  headersLoading: false,
+  selectedLogIndex: -1,
+};
+
+export const createLogsSlice = (
+  set: (fn: (state: StoreState) => void) => void,
+  get: () => StoreState,
+  _store: any,
+): [LogsSlice, () => void] => {
+  const logsPolling = createLogsPolling(get, set);
+
+  const slice = {
+    // State
+    logs: initialState,
+
+    // Actions
+    logsActions: {
+      setLogs: (logs: LogFiles) => {
+        set((state) => {
+          state.logs.logs = logs;
+        });
+
+        // If we have files in the logs, load the headers
+        if (logs.files.length > 0) {
+          // ensure state is updated first
+          setTimeout(() => {
+            const currentState = get();
+            if (!currentState.logs.headersLoading) {
+              logsPolling.startPolling(logs);
+            }
+          }, 100);
+        }
+      },
+      setLogHeaders: (headers: Record<string, EvalLogHeader>) =>
+        set((state) => {
+          state.logs.logHeaders = headers;
+        }),
+      setHeadersLoading: (loading: boolean) =>
+        set((state) => {
+          state.logs.headersLoading = loading;
+        }),
+      setSelectedLogIndex: (selectedLogIndex: number) => {
+        set((state) => {
+          state.logs.selectedLogIndex = selectedLogIndex;
+        });
+      },
+      updateLogHeaders: (headers: Record<string, EvalLogHeader>) =>
+        set((state) => {
+          state.logs.logHeaders = { ...get().logs.logHeaders, ...headers };
+        }),
+
+      setSelectedLogFile: (logUrl: string) => {
+        const state = get();
+        const index = state.logs.logs.files.findIndex((val) =>
+          logUrl.endsWith(val.name),
+        );
+
+        if (index > -1) {
+          state.logsActions.setSelectedLogIndex(index);
+        }
+      },
+
+      // Helper function to load logs
+      loadLogs: async () => {
+        const api = get().api;
+        if (!api) {
+          console.error("API not initialized in LogsStore");
+          return { log_dir: "", files: [] };
+        }
+
+        try {
+          log.debug("LOADING LOG FILES");
+          return await api.get_log_paths();
+        } catch (e) {
+          console.log(e);
+          get().appActions.setStatus({ loading: false, error: e as Error });
+          return { log_dir: "", files: [] };
+        }
+      },
+      refreshLogs: async () => {
+        log.debug("REFRESH LOGS");
+        const state = get();
+        const refreshedLogs = await state.logsActions.loadLogs();
+
+        // Set the logs first
+        state.logsActions.setLogs(refreshedLogs || { log_dir: "", files: [] });
+
+        // Preserve the selected log even if new logs appear
+        const currentLog =
+          refreshedLogs.files[
+            state.logs.selectedLogIndex > -1 ? state.logs.selectedLogIndex : 0
+          ];
+
+        if (currentLog) {
+          const newIndex = refreshedLogs?.files.findIndex((file) =>
+            currentLog.name.endsWith(file.name),
+          );
+
+          if (newIndex !== undefined && newIndex !== -1) {
+            state.logsActions.setSelectedLogIndex(newIndex);
+          }
+        }
+      },
+      // Select a specific log file
+      selectLogFile: async (logUrl: string) => {
+        const state = get();
+        const index = state.logs.logs.files.findIndex((val) =>
+          val.name.endsWith(logUrl),
+        );
+
+        // It is already loaded
+        if (index > -1) {
+          state.logsActions.setSelectedLogIndex(index);
+        } else {
+          // It isn't yet loaded, so refresh the logs and try to load it from there
+          const result = await state.logsActions.loadLogs();
+          const idx = result?.files.findIndex((file) =>
+            logUrl.endsWith(file.name),
+          );
+
+          state.logsActions.setLogs(result || { log_dir: "", files: [] });
+          state.logsActions.setSelectedLogIndex(
+            idx !== undefined && idx > -1 ? idx : 0,
+          );
+        }
+      },
+
+      getSelectedLogFile: () => {
+        const state = get();
+        const file = state.logs.logs.files[state.logs.selectedLogIndex];
+        return file !== undefined ? file.name : undefined;
+      },
+    },
+  } as const;
+
+  const cleanup = () => {};
+
+  return [slice, cleanup];
+};
+
+export const initializeLogsSlice = <T extends LogsSlice>(
+  set: (fn: (state: T) => void) => void,
+) => {
+  set((state) => {
+    if (!state.logs) {
+      state.logs = initialState;
+    }
+  });
+};