inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/samples/{tools/filters.mjs → sample-tools/filters.ts}

@@ -1,23 +1,31 @@
 import { compileExpression } from "filtrex";
-import { kScoreTypeBoolean } from "../../constants.mjs";
-import { inputString } from "../../utils/Format.mjs";
+import { SampleSummary } from "../../api/types";
+import { kScoreTypeBoolean } from "../../constants";
+import { ScoreLabel } from "../../types";
+import { Scores1 } from "../../types/log";
+import { inputString } from "../../utils/format";
+import { EvalDescriptor, ScoreDescriptor } from "../descriptor/types";
 
-/**
- * @typedef {Object} FilterError
- * @property {number=} from - The start of the error.
- * @property {number=} to - The end of the error.
- * @property {string} message - The error message.
- * @property {"warning" | "error"} severity - The severity of the error.
- */
+export interface FilterError {
+  from: number;
+  to: number;
+  message: string;
+  severity: "warning" | "error";
+}
+
+export interface ScoreFilterItem {
+  shortName?: string;
+  qualifiedName?: string;
+  canonicalName: string;
+  tooltip?: string;
+  categories: string[];
+  scoreType: string;
+}
 
 /**
  * Coerces a value to the type expected by the score.
- *
- * @param {any} value
- * @param {import("../../samples/SamplesDescriptor.mjs").ScoreDescriptor} descriptor
- * @returns {any}
  */
-const coerceValue = (value, descriptor) => {
+const coerceValue = (value: unknown, descriptor: ScoreDescriptor): unknown => {
   if (descriptor && descriptor.scoreType === kScoreTypeBoolean) {
     return Boolean(value);
   } else {
@@ -25,24 +33,18 @@ const coerceValue = (value, descriptor) => {
   }
 };
 
-/**
- * @param {any} value
- * @returns {boolean}
- */
-const isFilteringSupportedForValue = (value) =>
+// Whether a particular value is filter-able
+const isFilteringSupportedForValue = (value: unknown): boolean =>
   ["string", "number", "boolean"].includes(typeof value);
 
 /**
  * Returns the names of scores that are not allowed to be used as short names in
  * filter expressions because they are not unique. This should be applied only to
  * the nested scores, not to the top-level scorer names.
- *
- * @param {import("../../Types.mjs").ScoreLabel[]} scores
- * @returns {Set<string>}
  */
-const bannedShortScoreNames = (scores) => {
-  const used = new Set();
-  const banned = new Set();
+const bannedShortScoreNames = (scores: ScoreLabel[]): Set<string> => {
+  const used: Set<string> = new Set();
+  const banned: Set<string> = new Set();
   for (const { scorer, name } of scores) {
     banned.add(scorer);
     if (used.has(name)) {
@@ -60,19 +62,22 @@ const bannedShortScoreNames = (scores) => {
  * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
  * directly by name when it is unique.
  *
- * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
+ * @param {import("../../samples/descriptor/samplesDescriptor").EvalDescriptor} evalDescriptor
  * @param {import("../../types/log").Scores1} sampleScores
  * @returns {Object<string, any>}
  */
-const scoreVariables = (evalDescriptor, sampleScores) => {
+const scoreVariables = (
+  evalDescriptor: EvalDescriptor,
+  sampleScores: Scores1,
+) => {
   const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
-  const variables = {};
+  const variables: Record<string, unknown> = {};
 
-  /**
-   * @param {import("../../Types.mjs").ScoreLabel} scoreLabel
-   * @param {any} value
-   */
-  const addScore = (variableName, scoreLabel, value) => {
+  const addScore = (
+    variableName: string,
+    scoreLabel: ScoreLabel,
+    value: unknown,
+  ) => {
     const coercedValue = coerceValue(
       value,
       evalDescriptor.scoreDescriptor(scoreLabel),
@@ -82,7 +87,7 @@ const scoreVariables = (evalDescriptor, sampleScores) => {
     }
   };
 
-  for (const [scorer, score] of Object.entries(sampleScores)) {
+  for (const [scorer, score] of Object.entries(sampleScores || {})) {
     addScore(scorer, { scorer, name: scorer }, score.value);
     if (typeof score.value === "object") {
       for (const [name, value] of Object.entries(score.value)) {
@@ -96,39 +101,34 @@ const scoreVariables = (evalDescriptor, sampleScores) => {
   return variables;
 };
 
-/**
- * @typedef {Object} ScoreFilterItem
- * @property {string | undefined} shortName - The short name of the score, if doesn't conflict with other short names.
- * @property {string | undefined} qualifiedName - The `scorer.score` name for children of complex scorers.
- * @property {string} canonicalName - The canonical name: either `shortName` or `qualifiedName` (at least one must exist).
- * @property {string} tooltip - The informational tooltip for the score.
- * @property {string[]} categories - Category values for categorical scores.
- * @property {string} scoreType - The type of the score (e.g., 'numeric', 'categorical', 'boolean').
- */
-
 /**
  * Generates a dictionary of variables that can be used in the filter expression.
  * High-level scorer metrics can be accessed by name directly.
  * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
  * directly by name when it is unique.
- *
- * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
- * @returns {ScoreFilterItem[]}
  */
-export const scoreFilterItems = (evalDescriptor) => {
-  /** @type {ScoreFilterItem[]} */
-  const items = [];
+export const scoreFilterItems = (
+  evalDescriptor: EvalDescriptor,
+): ScoreFilterItem[] => {
+  const items: ScoreFilterItem[] = [];
   const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
-  const valueToString = (value) =>
+  const valueToString = (value: unknown) =>
     typeof value === "string" ? `"${value}"` : String(value);
 
   /**
    * @param {string | undefined} shortName
    * @param {string | undefined} qualifiedName
-   * @param {import("../../Types.mjs").ScoreLabel} scoreLabel
+   * @param {import("../../types").ScoreLabel} scoreLabel
    */
-  const addScore = (shortName, qualifiedName, scoreLabel) => {
+  const addScore = (
+    scoreLabel: ScoreLabel,
+    shortName?: string,
+    qualifiedName?: string,
+  ) => {
     const canonicalName = shortName || qualifiedName;
+    if (!canonicalName) {
+      throw new Error("Unable to create a canonical name for a score");
+    }
     const descriptor = evalDescriptor.scoreDescriptor(scoreLabel);
     const scoreType = descriptor?.scoreType;
     if (!descriptor) {
@@ -143,17 +143,20 @@ export const scoreFilterItems = (evalDescriptor) => {
       return;
     }
     var tooltip = `${canonicalName}: ${descriptor.scoreType}`;
-    var categories = [];
+    var categories: string[] = [];
     if (descriptor.min !== undefined || descriptor.max !== undefined) {
-      const rounded = (num) => {
+      const rounded = (num: number) => {
         // Additional round-trip to remove trailing zeros.
         return parseFloat(num.toPrecision(3)).toString();
       };
-      tooltip += `\nrange: ${rounded(descriptor.min)} to ${rounded(descriptor.max)}`;
+      tooltip += `\nrange: ${rounded(descriptor.min || 0)} to ${rounded(descriptor.max || 0)}`;
     }
     if (descriptor.categories) {
-      tooltip += `\ncategories: ${descriptor.categories.map((cat) => cat.val).join(", ")}`;
-      categories = descriptor.categories.map((cat) => valueToString(cat.val));
+      categories = descriptor.categories.map((cat) => {
+        const val = (cat as Record<string, unknown>).val;
+        return valueToString(val);
+      });
+      tooltip += `\ncategories: ${categories.join(" ")}`;
     }
     items.push({
       shortName,
@@ -170,29 +173,24 @@ export const scoreFilterItems = (evalDescriptor) => {
     const hasQualifiedName = name !== scorer;
     const shortName = hasShortName ? name : undefined;
     const qualifiedName = hasQualifiedName ? `${scorer}.${name}` : undefined;
-    addScore(shortName, qualifiedName, { name, scorer });
+    addScore({ name, scorer }, shortName, qualifiedName);
   }
   return items;
 };
 
-/**
- * TODO: Add case-insensitive string comparison.
- *
- * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
- * @param {import("../../api/Types.mjs").SampleSummary} sample
- * @param {string} filterValue
- * @returns {{matches: boolean, error: FilterError | undefined}}
- */
-export const filterExpression = (evalDescriptor, sample, filterValue) => {
+// TODO: Add case-insensitive string comparison.
+export const filterExpression = (
+  evalDescriptor: EvalDescriptor,
+  sample: SampleSummary,
+  filterValue: string,
+) => {
   try {
-    /** @type {(regex: string) => boolean} */
-    const inputContains = (regex) => {
+    const inputContains = (regex: string): boolean => {
       return inputString(sample.input).some((msg) =>
         msg.match(new RegExp(regex, "i")),
       );
     };
-    /** @type {(regex: string) => boolean} */
-    const targetContains = (regex) => {
+    const targetContains = (regex: string): boolean => {
       let targets = Array.isArray(sample.target)
         ? sample.target
         : [sample.target];
@@ -217,7 +215,8 @@ export const filterExpression = (evalDescriptor, sample, filterValue) => {
     }
   } catch (error) {
     if (error instanceof ReferenceError) {
-      const propertyName = error["propertyName"];
+      const errorObj = error as any as Record<string, unknown>;
+      const propertyName: string = (errorObj["propertyName"] as string) || "";
       if (propertyName) {
         const regex = new RegExp(`\\b${propertyName}\\b`);
         const match = regex.exec(filterValue);
@@ -234,14 +233,16 @@ export const filterExpression = (evalDescriptor, sample, filterValue) => {
         }
       }
     }
+
+    const message = error instanceof Error ? error.message : "";
     if (
-      error.message.startsWith("Parse error") ||
-      error.message.startsWith("Lexical error")
+      message.startsWith("Parse error") ||
+      message.startsWith("Lexical error")
     ) {
       // Filterex uses formatting like this:
       //   foo and
       //   ----^
-      const from = error.message.match(/^(-*)\^$/m)?.[1]?.length;
+      const from = message.match(/^(-*)\^$/m)?.[1]?.length;
       return {
         matches: false,
         error: {
@@ -255,7 +256,7 @@ export const filterExpression = (evalDescriptor, sample, filterValue) => {
     return {
       matches: false,
       error: {
-        message: error.message,
+        message: message,
         severity: "error",
       },
     };
@@ -263,12 +264,16 @@ export const filterExpression = (evalDescriptor, sample, filterValue) => {
 };
 
 /**
- * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
- * @param {import("../../api/Types.mjs").SampleSummary[]} samples
+ * @param {import("../../samples/descriptor/samplesDescriptor").EvalDescriptor} evalDescriptor
+ * @param {import("../../api/types").SampleSummary[]} samples
  * @param {string} filterValue
- * @returns {{result: import("../../api/Types.mjs").SampleSummary[], error: FilterError | undefined}}
+ * @returns {}
  */
-export const filterSamples = (evalDescriptor, samples, filterValue) => {
+export const filterSamples = (
+  evalDescriptor: EvalDescriptor,
+  samples: SampleSummary[],
+  filterValue: string,
+): { result: SampleSummary[]; error: FilterError | undefined } => {
   var error = undefined;
   const result = samples.filter((sample) => {
     if (filterValue) {

inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.module.css

@@ -0,0 +1,16 @@
+.label {
+  align-self: center;
+  margin-right: 0.3em;
+  margin-left: 0.2em;
+}
+
+.input {
+  width: 300px;
+}
+
+.help {
+  position: relative;
+  margin-left: 0.5em;
+  cursor: help;
+  align-self: center;
+}

inspect_ai/_view/www/src/samples/sample-tools/sample-filter/SampleFilter.tsx

@@ -0,0 +1,288 @@
+import { autocompletion, startCompletion } from "@codemirror/autocomplete";
+import {
+  bracketMatching,
+  HighlightStyle,
+  syntaxHighlighting,
+} from "@codemirror/language";
+import { Diagnostic, linter } from "@codemirror/lint";
+import {
+  Compartment,
+  EditorState,
+  Transaction,
+  TransactionSpec,
+} from "@codemirror/state";
+import { tags } from "@lezer/highlight";
+import clsx from "clsx";
+import { EditorView, minimalSetup } from "codemirror";
+import { useEffect, useMemo, useRef, useState } from "react";
+
+import { ScoreFilter } from "../../../types";
+import { EvalDescriptor } from "../../descriptor/types";
+import { FilterError, filterSamples, scoreFilterItems } from "../filters";
+import { getCompletions } from "./completions";
+import styles from "./SampleFilter.module.css";
+import { language } from "./tokenize";
+
+// Types
+interface FilteringResult {
+  numSamples: number;
+  error?: FilterError;
+}
+
+interface SampleFilterProps {
+  evalDescriptor: EvalDescriptor;
+  scoreFilter: ScoreFilter;
+  setScoreFilter: (filter: ScoreFilter) => void;
+}
+
+// Constants
+const FILTER_TOOLTIP = `
+Filter samples by:
+  • Scores
+  • Input and target regex search: input_contains, target_contains
+
+Supported expressions:
+  • Arithmetic: +, -, *, /, mod, ^
+  • Comparison: <, <=, >, >=, ==, !=, including chain comparisons, e.g. "10 <= x < 20"
+  • Boolean: and, or, not
+  • Regex matching: ~= (case-sensitive)
+  • Set operations: in, not in; e.g. "x in (2, 3, 5)"
+  • Functions: min, max, abs, round, floor, ceil, sqrt, log, log2, log10
+`.trim();
+
+// Styles
+const highlightStyle = HighlightStyle.define([
+  { tag: tags.string, class: "token string" },
+  { tag: tags.number, class: "token number" },
+  { tag: tags.keyword, class: "token keyword" },
+]);
+
+const editorTheme = EditorView.theme({
+  "&": {
+    fontSize: "inherit",
+    color: "var(--inspect-input-foreground)",
+    backgroundColor: "var(--inspect-input-background)",
+    border: "1px solid var(--inspect-input-border)",
+    borderRadius: "var(--bs-border-radius)",
+  },
+  ".cm-cursor.cm-cursor-primary": {
+    borderLeftColor: "var(--bs-body-color)",
+  },
+  ".cm-selectionBackground": {
+    backgroundColor: "var(--inspect-inactive-selection-background)",
+  },
+  "&.cm-focused > .cm-scroller > .cm-selectionLayer > .cm-selectionBackground":
+    {
+      backgroundColor: "var(--inspect-active-selection-background)",
+    },
+  "&.cm-focused": {
+    outline: "none",
+    borderColor: "var(--inspect-focus-border-color)",
+    boxShadow: "var(--inspect-focus-border-shadow)",
+  },
+  ".filter-pending > &.cm-focused": {
+    borderColor: "var(--inspect-focus-border-gray-color)",
+    boxShadow: "var(--inspect-focus-border-gray-shadow)",
+  },
+  ".cm-tooltip": {
+    backgroundColor: "var(--bs-light)",
+    border: "1px solid var(--bs-border-color)",
+    color: "var(--bs-body-color)",
+  },
+  ".cm-tooltip.cm-tooltip-autocomplete > ul > li": {
+    color: "var(--bs-body-color)",
+  },
+  ".cm-tooltip.cm-tooltip-autocomplete > ul > li[aria-selected]": {
+    backgroundColor: "var(--inspect-active-selection-background)",
+    color: "var(--bs-body-color)",
+  },
+  ".cm-scroller": {
+    overflow: "hidden",
+  },
+});
+
+// Helper functions
+const getFilteringResult = (
+  evalDescriptor: EvalDescriptor,
+  filterValue: string,
+): FilteringResult => {
+  const { result, error } = filterSamples(
+    evalDescriptor,
+    evalDescriptor.samples,
+    filterValue,
+  );
+  return { numSamples: result.length, error };
+};
+
+const ensureOneLine = (tr: Transaction): TransactionSpec => {
+  const newDoc = tr.newDoc.toString();
+  if (!newDoc.includes("\n")) return tr;
+
+  if (tr.isUserEvent("input.paste")) {
+    return {
+      changes: {
+        from: 0,
+        to: tr.startState.doc.length,
+        insert: newDoc.replace(/\n/g, " ").trim(),
+      },
+    };
+  }
+  return {};
+};
+
+const getLints = (
+  view: EditorView,
+  filterError?: FilterError,
+): Diagnostic[] => {
+  if (!filterError) return [];
+  return [
+    {
+      from: filterError.from || 0,
+      to: filterError.to || view.state.doc.length,
+      severity: filterError.severity,
+      message: filterError.message,
+    },
+  ];
+};
+
+// Main component
+export const SampleFilter: React.FC<SampleFilterProps> = ({
+  evalDescriptor,
+  scoreFilter,
+  setScoreFilter,
+}) => {
+  const editorRef = useRef<HTMLDivElement>(null);
+  const editorViewRef = useRef<EditorView>(null);
+  const linterCompartment = useRef<Compartment>(new Compartment());
+  const autocompletionCompartment = useRef<Compartment>(new Compartment());
+  const updateListenerCompartment = useRef<Compartment>(new Compartment());
+
+  const filterItems = useMemo(
+    () => scoreFilterItems(evalDescriptor),
+    [evalDescriptor],
+  );
+
+  const [filteringResultInstant, setFilteringResultInstant] =
+    useState<FilteringResult | null>(null);
+
+  const handleFocus = (event: FocusEvent, view: EditorView) => {
+    if (event.isTrusted && view.state.doc.toString() === "") {
+      setTimeout(() => startCompletion(view), 0);
+    }
+  };
+
+  const makeAutocompletion = () =>
+    autocompletion({
+      override: [(context) => getCompletions(context, filterItems)],
+      activateOnCompletion: (c) => c.label.endsWith(" "),
+    });
+
+  const makeLinter = () =>
+    linter((view) => getLints(view, filteringResultInstant?.error));
+
+  const makeUpdateListener = () =>
+    EditorView.updateListener.of((update) => {
+      if (update.docChanged) {
+        const newValue = update.state.doc.toString();
+        const filteringResult = getFilteringResult(evalDescriptor, newValue);
+        if (!filteringResult.error) {
+          setScoreFilter({ value: newValue });
+        }
+        setFilteringResultInstant(filteringResult);
+      }
+    });
+
+  // Initialize editor
+  useEffect(() => {
+    editorViewRef.current?.destroy();
+
+    editorViewRef.current = new EditorView({
+      parent: editorRef.current ?? undefined,
+      state: EditorState.create({
+        doc: scoreFilter.value || "",
+        extensions: [
+          minimalSetup,
+          bracketMatching(),
+          editorTheme,
+          EditorState.transactionFilter.of(ensureOneLine),
+          updateListenerCompartment.current.of(makeUpdateListener()),
+          EditorView.domEventHandlers({ focus: handleFocus }),
+          language,
+          syntaxHighlighting(highlightStyle),
+          autocompletionCompartment.current.of(makeAutocompletion()),
+          linterCompartment.current.of(makeLinter()),
+        ],
+      }),
+    });
+
+    return () => editorViewRef.current?.destroy();
+  }, []);
+
+  // Handle filter value changes
+  useEffect(() => {
+    if (!editorViewRef.current) return;
+
+    const currentValue = editorViewRef.current.state.doc.toString();
+    if (scoreFilter.value === currentValue) return;
+
+    setFilteringResultInstant(
+      getFilteringResult(evalDescriptor, scoreFilter.value || ""),
+    );
+    editorViewRef.current.dispatch({
+      changes: {
+        from: 0,
+        to: currentValue.length,
+        insert: scoreFilter.value || "",
+      },
+    });
+  }, [evalDescriptor, scoreFilter.value]);
+
+  // Update compartments when dependencies change
+  useEffect(() => {
+    editorViewRef.current?.dispatch({
+      effects:
+        updateListenerCompartment.current.reconfigure(makeUpdateListener()),
+    });
+  }, [evalDescriptor]);
+
+  useEffect(() => {
+    editorViewRef.current?.dispatch({
+      effects:
+        autocompletionCompartment.current.reconfigure(makeAutocompletion()),
+    });
+  }, [filterItems]);
+
+  useEffect(() => {
+    editorViewRef.current?.dispatch({
+      effects: linterCompartment.current.reconfigure(makeLinter()),
+    });
+  }, [filteringResultInstant?.error]);
+
+  return (
+    <div style={{ display: "flex" }}>
+      <span
+        className={clsx(
+          "sample-filter-label",
+          "text-size-smaller",
+          "text-style-label",
+          "text-style-secondary",
+          styles.label,
+        )}
+      >
+        Filter:
+      </span>
+      <div
+        ref={editorRef}
+        className={clsx(
+          filteringResultInstant?.error && "filter-pending",
+          styles.input,
+        )}
+      />
+      <span
+        className={clsx("bi", "bi-question-circle", styles.help)}
+        data-tooltip={FILTER_TOOLTIP}
+        data-tooltip-position="bottom-left"
+      />
+    </div>
+  );
+};