inspect-ai 0.3.94__py3-none-any.whl → 0.3.95__py3-none-any.whl

inspect_ai/_view/www/src/app/samples/sample-tools/SelectScorer.tsx

@@ -1,7 +1,7 @@
 import clsx from "clsx";
 import { ScoreLabel } from "../../../app/types";
 
-import { ChangeEvent, FC, useCallback } from "react";
+import { ChangeEvent, FC, useCallback, useMemo } from "react";
 import styles from "./SelectScorer.module.css";
 
 interface SelectScorerProps {
@@ -15,23 +15,18 @@ export const SelectScorer: FC<SelectScorerProps> = ({
   score,
   setScore,
 }) => {
-  const scorers = scores.reduce((accum, scorer) => {
-    if (
-      !accum.find((sc) => {
-        return scorer.scorer === sc.scorer;
-      })
-    ) {
-      accum.push(scorer);
-    }
-    return accum;
-  }, [] as ScoreLabel[]);
-
-  const handleSelectScore = useCallback(
-    (index: number) => {
-      setScore(scores[index]);
-    },
-    [setScore, scores],
-  );
+  const scorers = useMemo(() => {
+    return scores.reduce((accum, scorer) => {
+      if (
+        !accum.find((sc) => {
+          return scorer.scorer === sc.scorer;
+        })
+      ) {
+        accum.push(scorer);
+      }
+      return accum;
+    }, [] as ScoreLabel[]);
+  }, [scores]);
 
   if (scorers.length === 1) {
     // There is only a single scorer in play, just show the list of available scores
@@ -50,8 +45,8 @@ export const SelectScorer: FC<SelectScorerProps> = ({
         </span>
         <ScoreSelector
           scores={scores}
-          selectedIndex={scoreIndex(scores, score)}
-          setSelectedIndex={handleSelectScore}
+          selectedScore={score}
+          setSelectedScore={setScore}
         />
       </div>
     );
@@ -79,15 +74,15 @@ export const SelectScorer: FC<SelectScorerProps> = ({
         </span>
         <ScorerSelector
           scorers={scorers}
-          selectedIndex={scorerIndex(scorers, score)}
-          setSelectedIndex={handleSelectScore}
+          selectedScore={score}
+          setSelectedScore={setScore}
         />
         {scorerScores.length > 1 ? (
           <ScoreSelector
             className={clsx(styles.secondSel)}
             scores={scorerScores}
-            selectedIndex={scoreIndex(scorerScores, score)}
-            setSelectedIndex={handleSelectScore}
+            selectedScore={score}
+            setSelectedScore={setScore}
           />
         ) : undefined}
       </div>
@@ -97,25 +92,33 @@ export const SelectScorer: FC<SelectScorerProps> = ({
 
 interface ScoreSelectorProps {
   scores: ScoreLabel[];
-  selectedIndex: number;
-  setSelectedIndex: (index: number) => void;
+  selectedScore?: ScoreLabel;
+  setSelectedScore: (score: ScoreLabel) => void;
   className?: string | string[];
 }
 
 const ScoreSelector: FC<ScoreSelectorProps> = ({
   scores,
-  selectedIndex,
-  setSelectedIndex,
+  selectedScore,
+  setSelectedScore,
   className,
 }) => {
   const handleChange = useCallback(
     (e: ChangeEvent<HTMLSelectElement>) => {
       const sel = e.target as HTMLSelectElement;
-      setSelectedIndex(sel.selectedIndex);
+      setSelectedScore(scores[sel.selectedIndex]);
     },
-    [setSelectedIndex],
+    [setSelectedScore, scores],
   );
 
+  const index = scores.findIndex((sc) => {
+    return (
+      selectedScore &&
+      sc.name === selectedScore.name &&
+      sc.scorer === selectedScore.scorer
+    );
+  });
+
   return (
     <select
       className={clsx(
@@ -125,7 +128,7 @@ const ScoreSelector: FC<ScoreSelectorProps> = ({
         className,
       )}
       aria-label=".select-scorer-label"
-      value={scores[selectedIndex].name}
+      value={scores[index].name}
       onChange={handleChange}
     >
       {scores.map((score) => {
@@ -141,28 +144,32 @@ const ScoreSelector: FC<ScoreSelectorProps> = ({
 
 interface ScorerSelectorProps {
   scorers: ScoreLabel[];
-  selectedIndex: number;
-  setSelectedIndex: (index: number) => void;
+  selectedScore?: ScoreLabel;
+  setSelectedScore: (score: ScoreLabel) => void;
 }
 
 const ScorerSelector: FC<ScorerSelectorProps> = ({
   scorers,
-  selectedIndex,
-  setSelectedIndex,
+  selectedScore,
+  setSelectedScore,
 }) => {
   const handleChange = useCallback(
     (e: ChangeEvent<HTMLSelectElement>) => {
       const sel = e.target as HTMLSelectElement;
-      setSelectedIndex(sel.selectedIndex);
+      setSelectedScore(scorers[sel.selectedIndex]);
     },
-    [setSelectedIndex],
+    [setSelectedScore, scorers],
   );
 
+  const index = scorers.findIndex((sc) => {
+    return selectedScore && sc.scorer === selectedScore.scorer;
+  });
+
   return (
     <select
       className={clsx("form-select", "form-select-sm", "text-size-smaller")}
       aria-label=".epoch-filter-label"
-      value={scorers[selectedIndex].scorer}
+      value={scorers[index].scorer}
       onChange={handleChange}
     >
       {scorers.map((scorer) => {
@@ -175,13 +182,3 @@ const ScorerSelector: FC<ScorerSelectorProps> = ({
     </select>
   );
 };
-
-const scoreIndex = (scores: ScoreLabel[], score?: ScoreLabel) =>
-  scores.findIndex((sc) => {
-    return score && sc.name === score.name && sc.scorer === score.scorer;
-  });
-
-const scorerIndex = (scores: ScoreLabel[], score?: ScoreLabel) =>
-  scores.findIndex((sc) => {
-    return score && sc.scorer === score.scorer;
-  });

inspect_ai/_view/www/src/app/samples/sample-tools/filters.ts

@@ -1,19 +1,12 @@
 import { compileExpression } from "filtrex";
 import { Scores1 } from "../../../@types/log";
-import { ScoreLabel } from "../../../app/types";
+import { FilterError, ScoreLabel } from "../../../app/types";
 import { SampleSummary } from "../../../client/api/types";
 import { kScoreTypeBoolean } from "../../../constants";
 import { inputString } from "../../../utils/format";
 import { EvalDescriptor, ScoreDescriptor } from "../descriptor/types";
 
-export interface FilterError {
-  from: number;
-  to: number;
-  message: string;
-  severity: "warning" | "error";
-}
-
-export interface ScoreFilterItem {
+export interface SampleFilterItem {
   shortName?: string;
   qualifiedName?: string;
   canonicalName: string;
@@ -120,10 +113,10 @@ const sampleVariables = (sample: SampleSummary): Record<string, unknown> => {
  * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
  * directly by name when it is unique.
  */
-export const scoreFilterItems = (
+export const sampleFilterItems = (
   evalDescriptor: EvalDescriptor,
-): ScoreFilterItem[] => {
-  const items: ScoreFilterItem[] = [];
+): SampleFilterItem[] => {
+  const items: SampleFilterItem[] = [];
   const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
   const valueToString = (value: unknown) =>
     typeof value === "string" ? `"${value}"` : String(value);
@@ -296,8 +289,13 @@ export const filterSamples = (
   evalDescriptor: EvalDescriptor,
   samples: SampleSummary[],
   filterValue: string,
-): { result: SampleSummary[]; error: FilterError | undefined } => {
-  var error = undefined;
+): {
+  result: SampleSummary[];
+  error: FilterError | undefined;
+  allErrors: boolean;
+} => {
+  let error = undefined;
+  let errorCount = 0;
   const result = samples.filter((sample) => {
     if (filterValue) {
       const { matches, error: sampleError } = filterExpression(
@@ -306,10 +304,13 @@ export const filterSamples = (
         filterValue,
       );
       error ||= sampleError;
+      if (sampleError) {
+        errorCount++;
+      }
       return matches;
     } else {
       return true;
     }
   });
-  return { result, error };
+  return { result, error, allErrors: errorCount === samples.length };
 };

inspect_ai/_view/www/src/app/samples/sample-tools/sample-filter/SampleFilter.tsx

@@ -14,28 +14,18 @@ import {
 import { tags } from "@lezer/highlight";
 import clsx from "clsx";
 import { EditorView, minimalSetup } from "codemirror";
-import { FC, useEffect, useMemo, useRef, useState } from "react";
+import { FC, useCallback, useEffect, useMemo, useRef } from "react";
 
-import { ScoreFilter } from "../../../../app/types";
-import { SampleSummary } from "../../../../client/api/types";
 import { useEvalDescriptor } from "../../../../state/hooks";
-import { EvalDescriptor } from "../../descriptor/types";
-import { FilterError, filterSamples, scoreFilterItems } from "../filters";
+import { useStore } from "../../../../state/store";
+import { debounce } from "../../../../utils/sync";
+import { FilterError } from "../../../types";
+import { sampleFilterItems } from "../filters";
 import { getCompletions } from "./completions";
 import styles from "./SampleFilter.module.css";
 import { language } from "./tokenize";
 
-// Types
-interface FilteringResult {
-  numSamples: number;
-  error?: FilterError;
-}
-
-interface SampleFilterProps {
-  samples: SampleSummary[];
-  scoreFilter: ScoreFilter;
-  setScoreFilter: (filter: ScoreFilter) => void;
-}
+interface SampleFilterProps {}
 
 // Constants
 const FILTER_TOOLTIP = `
@@ -105,20 +95,6 @@ const editorTheme = EditorView.theme({
   },
 });
 
-// Helper functions
-const getFilteringResult = (
-  evalDescriptor: EvalDescriptor,
-  sampleSummaries: SampleSummary[],
-  filterValue: string,
-): FilteringResult => {
-  const { result, error } = filterSamples(
-    evalDescriptor,
-    sampleSummaries,
-    filterValue,
-  );
-  return { numSamples: result.length, error };
-};
-
 const ensureOneLine = (tr: Transaction): TransactionSpec => {
   const newDoc = tr.newDoc.toString();
   if (!newDoc.includes("\n")) return tr;
@@ -154,11 +130,7 @@ const getLints = (
 };
 
 // Main component
-export const SampleFilter: FC<SampleFilterProps> = ({
-  samples,
-  scoreFilter,
-  setScoreFilter,
-}) => {
+export const SampleFilter: FC<SampleFilterProps> = () => {
   const editorRef = useRef<HTMLDivElement>(null);
   const editorViewRef = useRef<EditorView>(null);
   const linterCompartment = useRef<Compartment>(new Compartment());
@@ -167,43 +139,51 @@ export const SampleFilter: FC<SampleFilterProps> = ({
   const evalDescriptor = useEvalDescriptor();
 
   const filterItems = useMemo(
-    () => (evalDescriptor ? scoreFilterItems(evalDescriptor) : []),
+    () => (evalDescriptor ? sampleFilterItems(evalDescriptor) : []),
     [evalDescriptor],
   );
 
-  const [filteringResultInstant, setFilteringResultInstant] =
-    useState<FilteringResult | null>(null);
+  const filter = useStore((state) => state.log.filter);
+  const filterError = useStore((state) => state.log.filterError);
+  const setFilter = useStore((state) => state.logActions.setFilter);
 
-  const handleFocus = (event: FocusEvent, view: EditorView) => {
+  const handleFocus = useCallback((event: FocusEvent, view: EditorView) => {
     if (event.isTrusted && view.state.doc.toString() === "") {
       setTimeout(() => startCompletion(view), 0);
     }
-  };
+  }, []);
 
-  const makeAutocompletion = () =>
-    autocompletion({
-      override: [(context) => getCompletions(context, filterItems)],
-      activateOnCompletion: (c) => c.label.endsWith(" "),
-    });
+  const makeAutocompletion = useCallback(
+    () =>
+      autocompletion({
+        override: [(context) => getCompletions(context, filterItems)],
+        activateOnCompletion: (c) => c.label.endsWith(" "),
+      }),
+    [],
+  );
 
-  const makeLinter = () =>
-    linter((view) => getLints(view, filteringResultInstant?.error));
+  const makeLinter = useCallback(
+    () => linter((view) => getLints(view, filterError)),
+    [filterError],
+  );
 
-  const makeUpdateListener = () =>
-    EditorView.updateListener.of((update) => {
-      if (update.docChanged && evalDescriptor) {
-        const newValue = update.state.doc.toString();
-        const filteringResult = getFilteringResult(
-          evalDescriptor,
-          samples,
-          newValue,
-        );
-        if (!filteringResult.error) {
-          setScoreFilter({ value: newValue });
+  const debounceSetFilter = useCallback(
+    debounce((value: string) => {
+      setFilter(value);
+    }, 200),
+    [setFilter],
+  );
+
+  const makeUpdateListener = useCallback(
+    () =>
+      EditorView.updateListener.of((update) => {
+        if (update.docChanged && evalDescriptor) {
+          const newValue = update.state.doc.toString();
+          debounceSetFilter(newValue);
         }
-        setFilteringResultInstant(filteringResult);
-      }
-    });
+      }),
+    [setFilter],
+  );
 
   // Initialize editor
   useEffect(() => {
@@ -212,7 +192,7 @@ export const SampleFilter: FC<SampleFilterProps> = ({
     editorViewRef.current = new EditorView({
       parent: editorRef.current ?? undefined,
       state: EditorState.create({
-        doc: scoreFilter.value || "",
+        doc: filter || "",
         extensions: [
           minimalSetup,
           bracketMatching(),
@@ -236,21 +216,16 @@ export const SampleFilter: FC<SampleFilterProps> = ({
     if (!editorViewRef.current) return;
 
     const currentValue = editorViewRef.current.state.doc.toString();
-    if (scoreFilter.value === currentValue) return;
+    if (filter === currentValue) return;
 
-    if (evalDescriptor) {
-      setFilteringResultInstant(
-        getFilteringResult(evalDescriptor, samples, scoreFilter.value || ""),
-      );
-    }
     editorViewRef.current.dispatch({
       changes: {
         from: 0,
         to: currentValue.length,
-        insert: scoreFilter.value || "",
+        insert: filter || "",
       },
     });
-  }, [evalDescriptor, scoreFilter.value]);
+  }, [filter]);
 
   // Update compartments when dependencies change
   useEffect(() => {
@@ -271,7 +246,7 @@ export const SampleFilter: FC<SampleFilterProps> = ({
     editorViewRef.current?.dispatch({
       effects: linterCompartment.current.reconfigure(makeLinter()),
     });
-  }, [filteringResultInstant?.error]);
+  }, [filterError]);
 
   return (
     <div style={{ display: "flex" }}>
@@ -288,10 +263,7 @@ export const SampleFilter: FC<SampleFilterProps> = ({
       </span>
       <div
         ref={editorRef}
-        className={clsx(
-          filteringResultInstant?.error && "filter-pending",
-          styles.input,
-        )}
+        className={clsx(filterError && "filter-pending", styles.input)}
       />
       <span
         className={clsx("bi", "bi-question-circle", styles.help)}

inspect_ai/_view/www/src/app/samples/sample-tools/sample-filter/completions.ts

@@ -12,7 +12,7 @@ import {
   kScoreTypeOther,
   kScoreTypePassFail,
 } from "../../../../constants";
-import { ScoreFilterItem } from "../filters";
+import { SampleFilterItem } from "../filters";
 import {
   KEYWORDS,
   MATH_FUNCTIONS,
@@ -29,7 +29,7 @@ interface CompletionOptions {
 }
 
 interface CanonicalNameCompletionProps {
-  autoSpaceIf?: (item: ScoreFilterItem) => boolean;
+  autoSpaceIf?: (item: SampleFilterItem) => boolean;
 }
 
 const isLiteral = (token: Token): boolean =>
@@ -98,7 +98,7 @@ const makeLiteralCompletion = (k: string): Completion => ({
 });
 
 const makeCanonicalNameCompletion = (
-  item: ScoreFilterItem,
+  item: SampleFilterItem,
   { autoSpaceIf = () => false }: CanonicalNameCompletionProps = {},
 ): Completion => ({
   label: item.canonicalName + (autoSpaceIf(item) ? " " : ""),
@@ -107,7 +107,7 @@ const makeCanonicalNameCompletion = (
   boost: 30,
 });
 
-const makeMemberAccessCompletion = (item: ScoreFilterItem): Completion => ({
+const makeMemberAccessCompletion = (item: SampleFilterItem): Completion => ({
   label: item.qualifiedName?.split(".")[1] || "",
   type: "variable",
   info: item.tooltip,
@@ -115,9 +115,9 @@ const makeMemberAccessCompletion = (item: ScoreFilterItem): Completion => ({
 });
 
 const getMemberScoreItems = (
-  filterItems: ScoreFilterItem[],
+  filterItems: SampleFilterItem[],
   scorer: string,
-): ScoreFilterItem[] =>
+): SampleFilterItem[] =>
   filterItems.filter((item) => item?.qualifiedName?.startsWith(`${scorer}.`));
 
 /**
@@ -136,7 +136,7 @@ const getMemberScoreItems = (
  */
 export function getCompletions(
   context: CompletionContext,
-  filterItems: ScoreFilterItem[],
+  filterItems: SampleFilterItem[],
 ): CompletionResult | null {
   const keywordCompletionItems = KEYWORDS.map(makeKeywordCompletion);
   const mathFunctionCompletionItems = MATH_FUNCTIONS.map(
@@ -177,7 +177,7 @@ export function getCompletions(
   const completionStart = currentToken ? currentToken.from : context.pos;
   const completingAtEnd = context.pos === doc.length;
 
-  const findFilterItem = (endIndex: number): ScoreFilterItem | undefined => {
+  const findFilterItem = (endIndex: number): SampleFilterItem | undefined => {
     if (prevToken(endIndex)?.type !== "variable") return undefined;
 
     let name = prevToken(endIndex).text;
@@ -267,7 +267,7 @@ export function getCompletions(
 
   const variableCompletions = () => makeCompletions(variableCompletionItems);
 
-  const memberAccessCompletions = (items: ScoreFilterItem[]) =>
+  const memberAccessCompletions = (items: SampleFilterItem[]) =>
     makeCompletions(items.map(makeMemberAccessCompletion), {
       autocompleteInTheMiddle: true,
       includeDefault: false,

inspect_ai/_view/www/src/app/types.ts

@@ -67,7 +67,9 @@ export interface LogState {
   selectedLogSummary?: EvalSummary;
   pendingSampleSummaries?: PendingSamples;
 
-  filter: ScoreFilter;
+  filter: string;
+  filterError?: FilterError;
+
   epoch: string;
   sort: string;
   score?: ScoreLabel;
@@ -122,8 +124,16 @@ export interface ScoreLabel {
   scorer: string;
 }
 
-export interface ScoreFilter {
+export interface SampleFilter {
   value?: string;
+  error?: FilterError;
+}
+
+export interface FilterError {
+  from: number;
+  to: number;
+  message: string;
+  severity: "warning" | "error";
 }
 
 export type SampleMode = "none" | "single" | "many";

inspect_ai/_view/www/src/components/ExpandablePanel.module.css

@@ -18,7 +18,7 @@
   display: flex;
   margin-top: 0;
   position: relative;
-  height: 8px;
+  height: 18px;
 }
 
 .moreToggle.bordered {

inspect_ai/_view/www/src/components/ExpandablePanel.tsx

@@ -27,19 +27,19 @@ export const ExpandablePanel: FC<ExpandablePanelProps> = memo(
     const [collapsed, setCollapsed] = useCollapsedState(id, collapse);
 
     const [showToggle, setShowToggle] = useState(false);
-    const lineHeightRef = useRef<number>(0);
+    const baseFontSizeRef = useRef<number>(0);
 
     const checkOverflow = useCallback(
       (entry: ResizeObserverEntry) => {
         const element = entry.target as HTMLDivElement;
 
         // Calculate line height if we haven't yet
-        if (!lineHeightRef.current) {
+        if (baseFontSizeRef.current === 0) {
           const computedStyle = window.getComputedStyle(element);
-          lineHeightRef.current = parseInt(computedStyle.lineHeight) || 16; // fallback to 16px if can't get line height
+          const rootFontSize = parseFloat(computedStyle.fontSize);
+          baseFontSizeRef.current = rootFontSize;
         }
-
-        const maxCollapsedHeight = lines * lineHeightRef.current;
+        const maxCollapsedHeight = baseFontSizeRef.current * lines;
         const contentHeight = element.scrollHeight;
 
         setShowToggle(contentHeight > maxCollapsedHeight);

inspect_ai/_view/www/src/state/hooks.ts

@@ -132,6 +132,11 @@ export const useFilteredSamples = () => {
   const evalDescriptor = useEvalDescriptor();
   const sampleSummaries = useSampleSummaries();
   const filter = useStore((state) => state.log.filter);
+  const setFilterError = useStore((state) => state.logActions.setFilterError);
+  const clearFilterError = useStore(
+    (state) => state.logActions.clearFilterError,
+  );
+
   const epoch = useStore((state) => state.log.epoch);
   const sort = useStore((state) => state.log.sort);
   const samplesDescriptor = useSampleDescriptor();
@@ -139,10 +144,19 @@ export const useFilteredSamples = () => {
 
   return useMemo(() => {
     // Apply filters
+    const { result, error, allErrors } =
+      evalDescriptor && filter
+        ? filterSamples(evalDescriptor, sampleSummaries, filter)
+        : { result: sampleSummaries, error: undefined, allErrors: false };
+
+    if (error && allErrors) {
+      setFilterError(error);
+    } else {
+      clearFilterError();
+    }
+
     const prefiltered =
-      evalDescriptor && filter.value
-        ? filterSamples(evalDescriptor, sampleSummaries, filter.value).result
-        : sampleSummaries;
+      error === undefined || !allErrors ? result : sampleSummaries;
 
     // Filter epochs
     const filtered =
@@ -160,6 +174,8 @@ export const useFilteredSamples = () => {
     evalDescriptor,
     sampleSummaries,
     filter,
+    setFilterError,
+    clearFilterError,
     epoch,
     sort,
     samplesDescriptor,