inspect-ai 0.3.81__py3-none-any.whl → 0.3.82__py3-none-any.whl

inspect_ai/_view/www/src/state/appSlice.ts

@@ -0,0 +1,244 @@
+import { StateSnapshot } from "react-virtuoso";
+import { Capabilities } from "../api/types";
+import { kEvalWorkspaceTabId, kSampleTranscriptTabId } from "../constants";
+import { AppState, AppStatus } from "../types";
+import { clearDocumentSelection } from "../utils/browser";
+import { StoreState } from "./store";
+
+export interface AppSlice {
+  app: AppState;
+  capabilities: Capabilities;
+  appActions: {
+    setStatus: (status: AppStatus) => void;
+    setOffcanvas: (show: boolean) => void;
+    setShowFind: (show: boolean) => void;
+    hideFind: () => void;
+
+    setShowingSampleDialog: (showing: boolean) => void;
+    setWorkspaceTab: (tab: string) => void;
+    clearWorkspaceTab: () => void;
+
+    setSampleTab: (tab: string) => void;
+    clearSampleTab: () => void;
+
+    getScrollPosition: (name: string) => number | undefined;
+    setScrollPosition: (name: string, value: number) => void;
+
+    getListPosition: (name: string) => StateSnapshot | undefined;
+    setListPosition: (name: string, state: StateSnapshot) => void;
+    clearListPosition: (name: string) => void;
+
+    getCollapsed: (name: string, defaultValue?: boolean) => boolean;
+    setCollapsed: (name: string, value: boolean) => void;
+
+    getMessageVisible: (name: string, defaultValue?: boolean) => boolean;
+    setMessageVisible: (name: string, value: boolean) => void;
+    clearMessageVisible: (name: string) => void;
+
+    getPropertyValue: <T>(bagName: string, key: string, defaultValue?: T) => T;
+    setPropertyValue: <T>(bagName: string, key: string, value: T) => void;
+    removePropertyValue: (bagName: string, key: string) => void;
+  };
+}
+
+const kDefaultWorkspaceTab = kEvalWorkspaceTabId;
+const kDefaultSampleTab = kSampleTranscriptTabId;
+
+const initialState: AppState = {
+  status: { loading: false },
+  offcanvas: false,
+  showFind: false,
+  dialogs: {
+    sample: false,
+  },
+  tabs: {
+    workspace: kDefaultWorkspaceTab,
+    sample: kDefaultSampleTab,
+  },
+  scrollPositions: {},
+  listPositions: {},
+  collapsed: {},
+  messages: {},
+  propertyBags: {},
+};
+
+export const createAppSlice = (
+  set: (fn: (state: StoreState) => void) => void,
+  get: () => StoreState,
+  _store: any,
+): [AppSlice, () => void] => {
+  const getBoolRecord = (
+    record: Record<string, boolean>,
+    name: string,
+    defaultValue?: boolean,
+  ) => {
+    if (Object.keys(record).includes(name)) {
+      return record[name];
+    } else {
+      return defaultValue || false;
+    }
+  };
+
+  const slice = {
+    // State
+    app: initialState,
+    capabilities: {} as Capabilities,
+
+    // Actions
+    appActions: {
+      setStatus: (status: AppStatus) =>
+        set((state) => {
+          state.app.status = status;
+        }),
+
+      setOffcanvas: (show: boolean) =>
+        set((state) => {
+          state.app.offcanvas = show;
+        }),
+
+      setShowFind: (show: boolean) =>
+        set((state) => {
+          state.app.showFind = show;
+        }),
+
+      hideFind: () => {
+        clearDocumentSelection();
+        set((state) => {
+          state.app.showFind = false;
+        });
+      },
+      setShowingSampleDialog: (showing: boolean) => {
+        set((state) => {
+          state.app.dialogs.sample = showing;
+        });
+        if (!showing) {
+          const state = get();
+          state.appActions.clearSampleTab();
+          state.sampleActions.clearSelectedSample();
+        }
+      },
+      setWorkspaceTab: (tab: string) => {
+        set((state) => {
+          state.app.tabs.workspace = tab;
+        });
+      },
+      clearWorkspaceTab: () => {
+        set((state) => {
+          state.app.tabs.workspace = kDefaultWorkspaceTab;
+        });
+      },
+      setSampleTab: (tab: string) => {
+        set((state) => {
+          state.app.tabs.sample = tab;
+        });
+      },
+      clearSampleTab: () => {
+        set((state) => {
+          state.app.tabs.sample = kDefaultSampleTab;
+        });
+      },
+      getScrollPosition: (name: string) => {
+        const state = get();
+        return state.app.scrollPositions[name];
+      },
+      setScrollPosition: (name: string, position: number) => {
+        set((state) => {
+          state.app.scrollPositions[name] = position;
+        });
+      },
+      getListPosition: (name: string) => {
+        const state = get();
+        if (Object.keys(state.app.listPositions).includes(name)) {
+          return state.app.listPositions[name];
+        } else {
+          return undefined;
+        }
+      },
+      setListPosition: (name: string, position: StateSnapshot) => {
+        set((state) => {
+          state.app.listPositions[name] = position;
+        });
+      },
+      clearListPosition: (name: string) => {
+        set((state) => {
+          // Remove the key
+          const newListPositions = { ...state.app.listPositions };
+          delete newListPositions[name];
+
+          return {
+            app: {
+              ...state.app,
+              listPositions: newListPositions,
+            },
+          };
+        });
+      },
+      getCollapsed: (name: string, defaultValue?: boolean) => {
+        return getBoolRecord(get().app.collapsed, name, defaultValue);
+      },
+      setCollapsed: (name: string, value: boolean) => {
+        set((state) => {
+          state.app.collapsed[name] = value;
+        });
+      },
+      getMessageVisible: (name: string, defaultValue?: boolean) => {
+        return getBoolRecord(get().app.messages, name, defaultValue);
+      },
+      setMessageVisible: (name: string, value: boolean) => {
+        set((state) => {
+          state.app.messages[name] = value;
+        });
+      },
+      clearMessageVisible: (name: string) => {
+        set((state) => {
+          delete state.app.messages[name];
+        });
+      },
+      getPropertyValue: <T>(
+        bagName: string,
+        key: string,
+        defaultValue?: T,
+      ): T => {
+        const state = get();
+        const bag = state.app.propertyBags[bagName] || {};
+        return (key in bag ? bag[key] : defaultValue) as T;
+      },
+
+      setPropertyValue: <T>(bagName: string, key: string, value: T) => {
+        set((state) => {
+          // Create the bag if it doesn't exist
+          if (!state.app.propertyBags[bagName]) {
+            state.app.propertyBags[bagName] = {};
+          }
+          // Only update the specific key
+          state.app.propertyBags[bagName][key] = value;
+        });
+      },
+
+      removePropertyValue: (bagName: string, key: string) => {
+        set((state) => {
+          if (state.app.propertyBags[bagName]) {
+            const { [key]: _, ...rest } = state.app.propertyBags[bagName];
+            state.app.propertyBags[bagName] = rest;
+          }
+        });
+      },
+    },
+  } as const;
+
+  const cleanup = () => {};
+
+  return [slice, cleanup];
+};
+
+export const initializeAppSlice = (
+  set: (fn: (state: StoreState) => void) => void,
+  capabilities: Capabilities,
+) => {
+  set((state) => {
+    state.capabilities = capabilities;
+    if (!state.app) {
+      state.app = initialState;
+    }
+  });
+};

inspect_ai/_view/www/src/state/hooks.ts

@@ -0,0 +1,397 @@
+import { highlightElement } from "prismjs";
+import { useCallback, useEffect, useMemo, useRef } from "react";
+import { SampleSummary } from "../api/types";
+import { kEpochAscVal, kSampleAscVal, kScoreAscVal } from "../constants";
+import {
+  createEvalDescriptor,
+  createSamplesDescriptor,
+} from "../samples/descriptor/samplesDescriptor";
+import { filterSamples } from "../samples/sample-tools/filters";
+import {
+  byEpoch,
+  bySample,
+  sortSamples,
+} from "../samples/sample-tools/SortFilter";
+import { getAvailableScorers, getDefaultScorer } from "../scoring/utils";
+import { Events } from "../types/log";
+import { createLogger } from "../utils/logger";
+import { useStore } from "./store";
+import { mergeSampleSummaries } from "./utils";
+
+const log = createLogger("hooks");
+
+export const useEvalSpec = () => {
+  const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
+  return selectedLogSummary?.eval;
+};
+
+// Fetches all samples summaries (both completed and incomplete)
+// without applying any filtering
+export const useSampleSummaries = () => {
+  const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
+  const pendingSampleSummaries = useStore(
+    (state) => state.log.pendingSampleSummaries,
+  );
+
+  return useMemo(() => {
+    return mergeSampleSummaries(
+      selectedLogSummary?.sampleSummaries || [],
+      pendingSampleSummaries?.samples || [],
+    );
+  }, [selectedLogSummary, pendingSampleSummaries]);
+};
+
+// Counts the total number of unfiltered sample summaries (both complete and incomplete)
+export const useTotalSampleCount = () => {
+  const sampleSummaries = useSampleSummaries();
+  return useMemo(() => {
+    return sampleSummaries.length;
+  }, [sampleSummaries]);
+};
+
+// Provides the currently selected score for this eval, providing a default
+// based upon the configuration (eval + summaries) if no scorer has been
+// selected
+export const useScore = () => {
+  const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
+  const sampleSummaries = useSampleSummaries();
+  const score = useStore((state) => state.log.score);
+  return useMemo(() => {
+    if (score) {
+      return score;
+    } else if (selectedLogSummary) {
+      return getDefaultScorer(selectedLogSummary, sampleSummaries);
+    } else {
+      return undefined;
+    }
+  }, [selectedLogSummary, sampleSummaries, score]);
+};
+
+// Provides the list of available scorers. Will inspect the eval or samples
+// to determine scores (even for in progress evals that don't yet have final
+// metrics)
+export const useScores = () => {
+  const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
+  const sampleSummaries = useSampleSummaries();
+  return useMemo(() => {
+    if (!selectedLogSummary) {
+      return [];
+    }
+
+    return getAvailableScorers(selectedLogSummary, sampleSummaries) || [];
+  }, [selectedLogSummary, sampleSummaries]);
+};
+
+// Provides the eval descriptor
+export const useEvalDescriptor = () => {
+  const scores = useScores();
+  const sampleSummaries = useSampleSummaries();
+  return useMemo(() => {
+    return scores ? createEvalDescriptor(scores, sampleSummaries) : null;
+  }, [scores, sampleSummaries]);
+};
+
+// Provides the sampls descriptor
+export const useSampleDescriptor = () => {
+  const evalDescriptor = useEvalDescriptor();
+  const sampleSummaries = useSampleSummaries();
+  const score = useScore();
+  return useMemo(() => {
+    return evalDescriptor
+      ? createSamplesDescriptor(sampleSummaries, evalDescriptor, score)
+      : undefined;
+  }, [evalDescriptor, sampleSummaries, score]);
+};
+
+// Provides the list of filtered samples
+// (applying sorting, grouping, and filtering)
+export const useFilteredSamples = () => {
+  const evalDescriptor = useEvalDescriptor();
+  const sampleSummaries = useSampleSummaries();
+  const filter = useStore((state) => state.log.filter);
+  const epoch = useStore((state) => state.log.epoch);
+  const sort = useStore((state) => state.log.sort);
+  const samplesDescriptor = useSampleDescriptor();
+  const score = useScore();
+
+  return useMemo(() => {
+    // Apply filters
+    const prefiltered =
+      evalDescriptor && filter.value
+        ? filterSamples(evalDescriptor, sampleSummaries, filter.value).result
+        : sampleSummaries;
+
+    // Filter epochs
+    const filtered =
+      epoch && epoch !== "all"
+        ? prefiltered.filter((sample) => epoch === String(sample.epoch))
+        : prefiltered;
+
+    // Sort samples
+    const sorted = samplesDescriptor
+      ? sortSamples(sort, filtered, samplesDescriptor, score)
+      : filtered;
+
+    return [...sorted];
+  }, [
+    evalDescriptor,
+    sampleSummaries,
+    filter,
+    epoch,
+    sort,
+    samplesDescriptor,
+    score,
+  ]);
+};
+
+// Computes the group by to use given a particular sort
+export const useGroupBy = () => {
+  const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
+  const sort = useStore((state) => state.log.sort);
+  const epoch = useStore((state) => state.log.epoch);
+  return useMemo(() => {
+    const epochs = selectedLogSummary?.eval?.config?.epochs || 1;
+    if (epochs > 1) {
+      if (byEpoch(sort) || epoch !== "all") {
+        return "epoch";
+      } else if (bySample(sort)) {
+        return "sample";
+      }
+    }
+
+    return "none";
+  }, [selectedLogSummary, sort, epoch]);
+};
+
+// Computes the ordering for groups based upon the sort
+export const useGroupByOrder = () => {
+  const sort = useStore((state) => state.log.sort);
+  return useMemo(() => {
+    return sort === kSampleAscVal ||
+      sort === kEpochAscVal ||
+      sort === kScoreAscVal
+      ? "asc"
+      : "desc";
+  }, [sort]);
+};
+
+// Provides the currently selected sample summary
+export const useSelectedSampleSummary = (): SampleSummary | undefined => {
+  const filteredSamples = useFilteredSamples();
+  const selectedIndex = useStore((state) => state.log.selectedSampleIndex);
+  return useMemo(() => {
+    return filteredSamples[selectedIndex];
+  }, [filteredSamples, selectedIndex]);
+};
+
+export const useSampleData = () => {
+  const sampleStatus = useStore((state) => state.sample.sampleStatus);
+  const sampleError = useStore((state) => state.sample.sampleError);
+  const selectedSample = useStore((state) => state.sample.selectedSample);
+  const runningEvents = useStore(
+    (state) => state.sample.runningEvents,
+  ) as Events;
+  return useMemo(() => {
+    return {
+      status: sampleStatus,
+      error: sampleError,
+      sample: selectedSample,
+      running: runningEvents,
+    };
+  }, [sampleStatus, sampleError, selectedSample, runningEvents]);
+};
+
+export const useLogSelection = () => {
+  const selectedSampleSummary = useSelectedSampleSummary();
+  const selectedLogFile = useStore((state) =>
+    state.logsActions.getSelectedLogFile(),
+  );
+
+  return useMemo(() => {
+    return {
+      logFile: selectedLogFile,
+      sample: selectedSampleSummary,
+    };
+  }, [selectedLogFile, selectedSampleSummary]);
+};
+
+export const useCollapsedState = (
+  id: string,
+  defaultValue?: boolean,
+): [boolean, (value: boolean) => void] => {
+  const collapsed = useStore((state) =>
+    state.appActions.getCollapsed(id, defaultValue),
+  );
+  const setCollapsed = useStore((state) => state.appActions.setCollapsed);
+  return useMemo(() => {
+    const set = (value: boolean) => {
+      log.debug("Set collapsed", id, value);
+      setCollapsed(id, value);
+    };
+    return [collapsed, set];
+  }, [collapsed, setCollapsed]);
+};
+
+export const useMessageVisibility = (
+  id: string,
+  scope: "sample" | "eval",
+): [boolean, (visible: boolean) => void] => {
+  const visible = useStore((state) =>
+    state.appActions.getMessageVisible(id, true),
+  );
+  const setVisible = useStore((state) => state.appActions.setMessageVisible);
+  const clearVisible = useStore(
+    (state) => state.appActions.clearMessageVisible,
+  );
+
+  // Track if this is the first render (rehydrate)
+  const isFirstRender = useRef(true);
+
+  // Reset state if the eval changes, but not during initialization
+  const selectedLogFile = useStore((state) =>
+    state.logsActions.getSelectedLogFile(),
+  );
+  useEffect(() => {
+    // Skip the first effect run
+    if (isFirstRender.current) {
+      isFirstRender.current = false;
+      return;
+    }
+
+    log.debug("clear message (eval)", id);
+    clearVisible(id);
+  }, [selectedLogFile, clearVisible, id]);
+
+  // Maybe reset state if sample changes
+  const selectedSampleIndex = useStore(
+    (state) => state.log.selectedSampleIndex,
+  );
+  useEffect(() => {
+    // Skip the first effect run for sample changes too
+    if (isFirstRender.current) {
+      return;
+    }
+
+    if (scope === "sample") {
+      log.debug("clear message (sample)", id);
+      clearVisible(id);
+    }
+  }, [selectedSampleIndex, clearVisible, id, scope]);
+
+  return useMemo(() => {
+    log.debug("visibility", id, visible);
+    const set = (visible: boolean) => {
+      log.debug("set visiblity", id);
+      setVisible(id, visible);
+    };
+    return [visible, set];
+  }, [visible, setVisible, id]);
+};
+
+export function useProperty<T>(
+  id: string,
+  propertyName: string,
+  options?: {
+    defaultValue?: T;
+    cleanup?: boolean;
+  },
+): [T, (value: T) => void, () => void] {
+  options = options || { cleanup: true };
+  const setPropertyValue = useStore(
+    (state) => state.appActions.setPropertyValue,
+  );
+  const removePropertyValue = useStore(
+    (state) => state.appActions.removePropertyValue,
+  );
+  const propertyValue = useStore(
+    useCallback(
+      (state) =>
+        state.appActions.getPropertyValue(
+          id,
+          propertyName,
+          options.defaultValue,
+        ),
+      [id, propertyName, options.defaultValue],
+    ),
+  );
+
+  const setValue = useCallback(
+    (value: T) => {
+      setPropertyValue(id, propertyName, value);
+    },
+    [id, propertyName, setPropertyValue],
+  );
+
+  const removeValue = useCallback(() => {
+    removePropertyValue(id, propertyName);
+  }, [id, propertyName, removePropertyValue]);
+
+  // Clean up on unmount
+  useEffect(() => {
+    return () => {
+      if (options.cleanup) {
+        removePropertyValue(id, propertyName);
+      }
+    };
+  }, [id, propertyName, removePropertyValue]);
+
+  return [propertyValue, setValue, removeValue];
+}
+
+export const usePrevious = <T>(value: T) => {
+  const ref = useRef<T | undefined>(undefined);
+
+  useEffect(() => {
+    ref.current = value;
+  }, [value]);
+
+  return ref.current;
+};
+
+// Syntax highlighting strings larger than this is too slow
+const kPrismRenderMaxSize = 250000;
+
+export const usePrismHighlight = (toolCallContent?: string) => {
+  const toolViewRef = useRef<HTMLDivElement>(null);
+
+  useEffect(() => {
+    if (
+      toolCallContent &&
+      toolViewRef.current &&
+      toolCallContent.length <= kPrismRenderMaxSize
+    ) {
+      requestAnimationFrame(() => {
+        const codeBlocks = toolViewRef.current!.querySelectorAll("pre code");
+        codeBlocks.forEach((block) => {
+          if (block.className.includes("language-")) {
+            block.classList.add("sourceCode");
+            highlightElement(block as HTMLElement);
+          }
+        });
+      });
+    }
+  }, [toolCallContent]);
+
+  return toolViewRef;
+};
+
+export const useSetSelectedLogIndex = () => {
+  const setSelectedLogIndex = useStore(
+    (state) => state.logsActions.setSelectedLogIndex,
+  );
+  const clearSelectedSample = useStore(
+    (state) => state.sampleActions.clearSelectedSample,
+  );
+  const clearSelectedLogSummary = useStore(
+    (state) => state.logActions.clearSelectedLogSummary,
+  );
+
+  return useCallback(
+    (index: number) => {
+      clearSelectedSample();
+      clearSelectedLogSummary();
+      setSelectedLogIndex(index);
+    },
+    [setSelectedLogIndex, clearSelectedLogSummary, clearSelectedSample],
+  );
+};