inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl

inspect_ai/_view/www/src/samples/tools/SortFilter.mjs

@@ -89,9 +89,9 @@ const sortId = (a, b) => {
  * Sorts a list of samples
  *
  * @param {string} sort - The sort direction
- * @param {import("../../api/Types.mjs").SampleSummary[]} samples - The samples
+ * @param {import("../../api/Types.ts").SampleSummary[]} samples - The samples
  * @param {import("../SamplesDescriptor.mjs").SamplesDescriptor} samplesDescriptor - The samples descriptor
- * @returns {{ sorted: import("../../api/Types.mjs").SampleSummary[], order: 'asc' | 'desc' }} An object with sorted samples and the sort order.
+ * @returns {{ sorted: import("../../api/Types.ts").SampleSummary[], order: 'asc' | 'desc' }} An object with sorted samples and the sort order.
  */
 export const sortSamples = (sort, samples, samplesDescriptor) => {
   const sortedSamples = samples.sort((a, b) => {

inspect_ai/_view/www/src/samples/tools/filters.mjs

@@ -1,114 +1,287 @@
-import { kScoreTypeCategorical, kScoreTypeNumeric } from "../../constants.mjs";
-import { isNumeric } from "../../utils/Type.mjs";
+import { compileExpression } from "filtrex";
+import { kScoreTypeBoolean } from "../../constants.mjs";
+import { inputString } from "../../utils/Format.mjs";
 
 /**
- * Gets a filter function for the specified type
+ * @typedef {Object} FilterError
+ * @property {number=} from - The start of the error.
+ * @property {number=} to - The end of the error.
+ * @property {string} message - The error message.
+ * @property {"warning" | "error"} severity - The severity of the error.
+ */
+
+/**
+ * Coerces a value to the type expected by the score.
  *
- * @param {import("../../Types.mjs").ScoreFilter} filter - The parameters for the component.
- * @returns {(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean | undefined} the function
+ * @param {any} value
+ * @param {import("../../samples/SamplesDescriptor.mjs").ScoreDescriptor} descriptor
+ * @returns {any}
  */
-export const filterFnForType = (filter) => {
-  if (filter.type) {
-    return filterFnsForType[filter.type];
+const coerceValue = (value, descriptor) => {
+  if (descriptor && descriptor.scoreType === kScoreTypeBoolean) {
+    return Boolean(value);
   } else {
-    return undefined;
+    return value;
   }
 };
 
 /**
- * @type{(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean}
+ * @param {any} value
+ * @returns {boolean}
  */
-const filterCategory = (descriptor, sample, value) => {
-  const score = descriptor.selectedScore(sample);
-  if (typeof score.value === "string") {
-    return score.value.toLowerCase() === value?.toLowerCase();
-  } else if (typeof score.value === "object") {
-    return JSON.stringify(score.value) == value;
-  } else {
-    return String(score.value) === value;
+const isFilteringSupportedForValue = (value) =>
+  ["string", "number", "boolean"].includes(typeof value);
+
+/**
+ * Returns the names of scores that are not allowed to be used as short names in
+ * filter expressions because they are not unique. This should be applied only to
+ * the nested scores, not to the top-level scorer names.
+ *
+ * @param {import("../../Types.mjs").ScoreLabel[]} scores
+ * @returns {Set<string>}
+ */
+const bannedShortScoreNames = (scores) => {
+  const used = new Set();
+  const banned = new Set();
+  for (const { scorer, name } of scores) {
+    banned.add(scorer);
+    if (used.has(name)) {
+      banned.add(name);
+    } else {
+      used.add(name);
+    }
   }
+  return banned;
 };
 
 /**
- * @type{(descriptor: import("../SamplesDescriptor.mjs").SamplesDescriptor, sample: import("../../api/Types.mjs").SampleSummary, value: string) => boolean}
+ * Generates a dictionary of variables that can be used in the filter expression.
+ * High-level scorer metrics can be accessed by name directly.
+ * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
+ * directly by name when it is unique.
+ *
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
+ * @param {import("../../types/log").Scores1} sampleScores
+ * @returns {Object<string, any>}
  */
-const filterText = (descriptor, sample, value) => {
-  const score = descriptor.selectedScore(sample);
-  if (!value) {
-    return true;
-  } else {
-    if (isNumeric(value)) {
-      if (typeof score.value === "number") {
-        return score.value === Number(value);
-      } else {
-        return Number(score.value) === Number(value);
-      }
-    } else {
-      const filters = [
-        {
-          prefix: ">=",
-          fn: (score, val) => {
-            return score >= val;
-          },
-        },
-        {
-          prefix: "<=",
-          fn: (score, val) => {
-            return score <= val;
-          },
-        },
-        {
-          prefix: ">",
-          fn: (score, val) => {
-            return score > val;
-          },
-        },
-        {
-          prefix: "<",
-          fn: (score, val) => {
-            return score < val;
-          },
-        },
-        {
-          prefix: "=",
-          fn: (score, val) => {
-            return score === val;
-          },
-        },
-        {
-          prefix: "!=",
-          fn: (score, val) => {
-            return score !== val;
-          },
-        },
-      ];
+const scoreVariables = (evalDescriptor, sampleScores) => {
+  const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
+  const variables = {};
 
-      for (const filter of filters) {
-        if (value?.startsWith(filter.prefix)) {
-          const val = value.slice(filter.prefix.length).trim();
-          if (!val) {
-            return true;
-          }
+  /**
+   * @param {import("../../Types.mjs").ScoreLabel} scoreLabel
+   * @param {any} value
+   */
+  const addScore = (variableName, scoreLabel, value) => {
+    const coercedValue = coerceValue(
+      value,
+      evalDescriptor.scoreDescriptor(scoreLabel),
+    );
+    if (isFilteringSupportedForValue(coercedValue)) {
+      variables[variableName] = coercedValue;
+    }
+  };
 
-          const num = Number(val);
-          return filter.fn(score.value, num);
+  for (const [scorer, score] of Object.entries(sampleScores)) {
+    addScore(scorer, { scorer, name: scorer }, score.value);
+    if (typeof score.value === "object") {
+      for (const [name, value] of Object.entries(score.value)) {
+        addScore(`${scorer}.${name}`, { scorer, name }, value);
+        if (!bannedShortNames.has(name)) {
+          addScore(name, { scorer, name }, value);
         }
       }
-      if (typeof score.value === "string") {
-        return score.value.toLowerCase() === value?.toLowerCase();
-      } else {
-        return String(score.value) === value;
-      }
     }
   }
+  return variables;
+};
+
+/**
+ * @typedef {Object} ScoreFilterItem
+ * @property {string | undefined} shortName - The short name of the score, if doesn't conflict with other short names.
+ * @property {string | undefined} qualifiedName - The `scorer.score` name for children of complex scorers.
+ * @property {string} canonicalName - The canonical name: either `shortName` or `qualifiedName` (at least one must exist).
+ * @property {string} tooltip - The informational tooltip for the score.
+ * @property {string[]} categories - Category values for categorical scores.
+ * @property {string} scoreType - The type of the score (e.g., 'numeric', 'categorical', 'boolean').
+ */
+
+/**
+ * Generates a dictionary of variables that can be used in the filter expression.
+ * High-level scorer metrics can be accessed by name directly.
+ * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
+ * directly by name when it is unique.
+ *
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
+ * @returns {ScoreFilterItem[]}
+ */
+export const scoreFilterItems = (evalDescriptor) => {
+  /** @type {ScoreFilterItem[]} */
+  const items = [];
+  const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
+  const valueToString = (value) =>
+    typeof value === "string" ? `"${value}"` : String(value);
+
+  /**
+   * @param {string | undefined} shortName
+   * @param {string | undefined} qualifiedName
+   * @param {import("../../Types.mjs").ScoreLabel} scoreLabel
+   */
+  const addScore = (shortName, qualifiedName, scoreLabel) => {
+    const canonicalName = shortName || qualifiedName;
+    const descriptor = evalDescriptor.scoreDescriptor(scoreLabel);
+    const scoreType = descriptor?.scoreType;
+    if (!descriptor) {
+      items.push({
+        shortName,
+        qualifiedName,
+        canonicalName,
+        tooltip: undefined,
+        categories: [],
+        scoreType,
+      });
+      return;
+    }
+    var tooltip = `${canonicalName}: ${descriptor.scoreType}`;
+    var categories = [];
+    if (descriptor.min !== undefined || descriptor.max !== undefined) {
+      const rounded = (num) => {
+        // Additional round-trip to remove trailing zeros.
+        return parseFloat(num.toPrecision(3)).toString();
+      };
+      tooltip += `\nrange: ${rounded(descriptor.min)} to ${rounded(descriptor.max)}`;
+    }
+    if (descriptor.categories) {
+      tooltip += `\ncategories: ${descriptor.categories.map((cat) => cat.val).join(", ")}`;
+      categories = descriptor.categories.map((cat) => valueToString(cat.val));
+    }
+    items.push({
+      shortName,
+      qualifiedName,
+      canonicalName,
+      tooltip,
+      categories,
+      scoreType,
+    });
+  };
+
+  for (const { name, scorer } of evalDescriptor.scores) {
+    const hasShortName = name === scorer || !bannedShortNames.has(name);
+    const hasQualifiedName = name !== scorer;
+    const shortName = hasShortName ? name : undefined;
+    const qualifiedName = hasQualifiedName ? `${scorer}.${name}` : undefined;
+    addScore(shortName, qualifiedName, { name, scorer });
+  }
+  return items;
 };
 
 /**
- * A dictionary that maps filter types to their respective filter functions.
+ * TODO: Add case-insensitive string comparison.
  *
- * @type {Record<string, (descriptor, sample, value) => boolean>}
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
+ * @param {import("../../api/Types.mjs").SampleSummary} sample
+ * @param {string} filterValue
+ * @returns {{matches: boolean, error: FilterError | undefined}}
+ */
+export const filterExpression = (evalDescriptor, sample, filterValue) => {
+  try {
+    /** @type {(regex: string) => boolean} */
+    const inputContains = (regex) => {
+      return inputString(sample.input).some((msg) =>
+        msg.match(new RegExp(regex, "i")),
+      );
+    };
+    /** @type {(regex: string) => boolean} */
+    const targetContains = (regex) => {
+      let targets = Array.isArray(sample.target)
+        ? sample.target
+        : [sample.target];
+      return targets.some((target) => target.match(new RegExp(regex, "i")));
+    };
+
+    const extraFunctions = {
+      input_contains: inputContains,
+      target_contains: targetContains,
+    };
+    const expression = compileExpression(filterValue, { extraFunctions });
+    const vars = scoreVariables(evalDescriptor, sample.scores);
+    const result = expression(vars);
+    if (typeof result === "boolean") {
+      return { matches: result, error: undefined };
+    } else if (result instanceof Error) {
+      throw result;
+    } else {
+      throw new TypeError(
+        `Filter expression returned a non-boolean value: ${result}`,
+      );
+    }
+  } catch (error) {
+    if (error instanceof ReferenceError) {
+      const propertyName = error["propertyName"];
+      if (propertyName) {
+        const regex = new RegExp(`\\b${propertyName}\\b`);
+        const match = regex.exec(filterValue);
+        if (match) {
+          return {
+            matches: false,
+            error: {
+              from: match.index,
+              to: match.index + propertyName.length,
+              message: error.message,
+              severity: "warning",
+            },
+          };
+        }
+      }
+    }
+    if (
+      error.message.startsWith("Parse error") ||
+      error.message.startsWith("Lexical error")
+    ) {
+      // Filterex uses formatting like this:
+      //   foo and
+      //   ----^
+      const from = error.message.match(/^(-*)\^$/m)?.[1]?.length;
+      return {
+        matches: false,
+        error: {
+          from: from,
+          message: "Syntax error",
+          severity: "error",
+        },
+      };
+    }
+
+    return {
+      matches: false,
+      error: {
+        message: error.message,
+        severity: "error",
+      },
+    };
+  }
+};
+
+/**
+ * @param {import("../../samples/SamplesDescriptor.mjs").EvalDescriptor} evalDescriptor
+ * @param {import("../../api/Types.mjs").SampleSummary[]} samples
+ * @param {string} filterValue
+ * @returns {{result: import("../../api/Types.mjs").SampleSummary[], error: FilterError | undefined}}
  */
-const filterFnsForType = {
-  [kScoreTypeCategorical]: filterCategory,
-  [kScoreTypeNumeric]: filterText,
+export const filterSamples = (evalDescriptor, samples, filterValue) => {
+  var error = undefined;
+  const result = samples.filter((sample) => {
+    if (filterValue) {
+      const { matches, error: sampleError } = filterExpression(
+        evalDescriptor,
+        sample,
+        filterValue,
+      );
+      error ||= sampleError;
+      return matches;
+    } else {
+      return true;
+    }
+  });
+  return { result, error };
 };

inspect_ai/_view/www/src/samples/transcript/ErrorEventView.mjs

@@ -12,11 +12,33 @@ import { formatDateTime } from "../../utils/Format.mjs";
  * @param { string  } props.id - The id of this event.
  * @param {import("../../types/log").ErrorEvent} props.event - The event object to display.
  * @param { Object } props.style - The style of this event.
+ * @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
+ * @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
  * @returns {import("preact").JSX.Element} The component.
  */
-export const ErrorEventView = ({ id, event, style }) => {
+export const ErrorEventView = ({
+  id,
+  event,
+  style,
+  eventState,
+  setEventState,
+}) => {
   return html`
-  <${EventPanel} id=${id} title="Error" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.error} style=${style}>
+  <${EventPanel} 
+    id=${id} 
+    title="Error" 
+    subTitle=${formatDateTime(new Date(event.timestamp))} 
+    icon=${ApplicationIcons.error} 
+    style=${style}
+    selectedNav=${eventState.selectedNav || ""}
+    onSelectedNav=${(selectedNav) => {
+      setEventState({ ...eventState, selectedNav });
+    }}
+    collapsed=${eventState.collapsed}
+    onCollapsed=${(collapsed) => {
+      setEventState({ ...eventState, collapsed });
+    }}
+  >
     <${ANSIDisplay} output=${event.error.traceback_ansi} style=${{ fontSize: "clamp(0.5rem, calc(0.25em + 1vw), 0.8rem)", margin: "0.5em 0" }}/>
   </${EventPanel}>`;
 };

inspect_ai/_view/www/src/samples/transcript/EventPanel.mjs

@@ -1,6 +1,5 @@
 // @ts-check
 import { html } from "htm/preact";
-import { useEffect, useMemo, useState } from "preact/hooks";
 import { ApplicationIcons } from "../../appearance/Icons.mjs";
 import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
 
@@ -16,8 +15,12 @@ import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
  * @param {string | undefined} props.icon - The icon of the event
  * @param {number | undefined} props.titleColor - The title color of this item
  * @param {boolean | undefined} props.collapse - Default collapse behavior for card. If omitted, not collapsible.
+ * @param {(collapse: boolean) => void} props.onCollapsed - handler for when collapsing
+ * @param {boolean} props.collapsed - handler for when collapsing
  * @param {Object} props.style - The style properties passed to the component.
  * @param {Object} props.titleStyle - The style properties passed to the title component.
+ * @param {(nav: string) => void} props.onSelectedNav - Handler for selected nav changed
+ * @param {string} props.selectedNav - The selected nav for this panel
  * @param {import("preact").ComponentChildren} props.children - The rendered event.
  * @returns {import("preact").JSX.Element} The component.
  */
@@ -30,22 +33,16 @@ export const EventPanel = ({
   icon,
   titleColor,
   collapse,
+  collapsed,
+  onCollapsed,
   style,
   titleStyle,
   children,
+  onSelectedNav,
+  selectedNav,
 }) => {
   const hasCollapse = collapse !== undefined;
-  const [collapsed, setCollapsed] = useState(!!collapse);
-  const [selectedNav, setSelectedNav] = useState("");
-
-  const filteredArrChildren = useMemo(() => {
-    const arrChildren = Array.isArray(children) ? children : [children];
-    return arrChildren.filter((child) => !!child);
-  }, [children]);
-
-  useEffect(() => {
-    setSelectedNav(pillId(0));
-  }, [filteredArrChildren]);
+  const isCollapsed = collapsed === undefined ? collapse : collapsed;
 
   /**
    * Generates the id for the navigation pill.
@@ -57,6 +54,11 @@ export const EventPanel = ({
     return `${id}-nav-pill-${index}`;
   };
 
+  const filteredArrChildren = (
+    Array.isArray(children) ? children : [children]
+  ).filter((child) => !!child);
+  const defaultPillId = pillId(0);
+
   const gridColumns = [];
   if (hasCollapse) {
     gridColumns.push("minmax(0, max-content)");
@@ -87,9 +89,9 @@ export const EventPanel = ({
           ${hasCollapse
             ? html`<i
                 onclick=${() => {
-                  setCollapsed(!collapsed);
+                  onCollapsed(!isCollapsed);
                 }}
-                class=${collapsed
+                class=${isCollapsed
                   ? ApplicationIcons.chevron.right
                   : ApplicationIcons.chevron.down}
               />`
@@ -103,7 +105,7 @@ export const EventPanel = ({
                   ...titleStyle,
                 }}
                 onclick=${() => {
-                  setCollapsed(!collapsed);
+                  onCollapsed(!isCollapsed);
                 }}
               />`
             : ""}
@@ -115,14 +117,14 @@ export const EventPanel = ({
               ...titleStyle,
             }}
             onclick=${() => {
-              setCollapsed(!collapsed);
+              onCollapsed(!isCollapsed);
             }}
           >
             ${title}
           </div>
           <div
             onclick=${() => {
-              setCollapsed(!collapsed);
+              onCollapsed(!isCollapsed);
             }}
           ></div>
           <div
@@ -132,7 +134,7 @@ export const EventPanel = ({
               marginRight: "0.2em",
             }}
             onclick=${() => {
-              setCollapsed(!collapsed);
+              onCollapsed(!isCollapsed);
             }}
           >
             ${collapsed ? text : ""}
@@ -144,7 +146,7 @@ export const EventPanel = ({
               flexDirection: "columns",
             }}
           >
-            ${(!hasCollapse || !collapsed) &&
+            ${(!hasCollapse || !isCollapsed) &&
             filteredArrChildren &&
             filteredArrChildren.length > 1
               ? html` <${EventNavs}
@@ -160,8 +162,8 @@ export const EventPanel = ({
                       target: pillId(index),
                     };
                   })}
-                  selectedNav=${selectedNav}
-                  setSelectedNav=${setSelectedNav}
+                  selectedNav=${selectedNav || defaultPillId}
+                  setSelectedNav=${onSelectedNav}
                 />`
               : ""}
           </div>
@@ -172,7 +174,6 @@ export const EventPanel = ({
     id=${id}
     style=${{
       padding: "0.625rem",
-      marginBottom: "0.625rem",
       border: "solid 1px var(--bs-light-border-subtle)",
       borderRadius: "var(--bs-border-radius)",
       ...style,
@@ -184,14 +185,18 @@ export const EventPanel = ({
       class="tab-content"
       style=${{
         padding: "0",
-        display: !hasCollapse || !collapsed ? "inherit" : "none",
+        display: !hasCollapse || !isCollapsed ? "inherit" : "none",
       }}
     >
       ${filteredArrChildren?.map((child, index) => {
         const id = pillId(index);
+        const isSelected = selectedNav
+          ? id === selectedNav
+          : id === defaultPillId;
+
         return html`<div
           id=${id}
-          class="tab-pane show ${id === selectedNav ? "active" : ""}"
+          class="tab-pane show ${isSelected ? "active" : ""}"
         >
           ${child}
         </div>`;

inspect_ai/_view/www/src/samples/transcript/EventRow.mjs

@@ -34,7 +34,7 @@ export const EventRow = ({ title, icon, style, children }) => {
     class="card"
     style=${{
       padding: "0.4em",
-      marginBottom: "0.4em",
+      marginBottom: "0",
       border: "solid 1px var(--bs-light-border-subtle)",
       borderRadius: "var(--bs-border-radius)",
       ...style,

inspect_ai/_view/www/src/samples/transcript/InfoEventView.mjs

@@ -13,9 +13,17 @@ import { formatDateTime } from "../../utils/Format.mjs";
  * @param { string  } props.id - The id of this event.
  * @param { Object } props.style - The depth of this event.
  * @param {import("../../types/log").InfoEvent} props.event - The event object to display.
+ * @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
+ * @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
  * @returns {import("preact").JSX.Element} The component.
  */
-export const InfoEventView = ({ id, event, style }) => {
+export const InfoEventView = ({
+  id,
+  event,
+  style,
+  eventState,
+  setEventState,
+}) => {
   const panels = [];
   if (typeof event.data === "string") {
     panels.push(
@@ -31,7 +39,21 @@ export const InfoEventView = ({ id, event, style }) => {
   }
 
   return html`
-  <${EventPanel} id=${id} title="Info" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.info} style=${style}>
+  <${EventPanel} 
+    id=${id} 
+    title="Info" 
+    subTitle=${formatDateTime(new Date(event.timestamp))} 
+    icon=${ApplicationIcons.info} 
+    style=${style}
+    selectedNav=${eventState.selectedNav || ""}
+    onSelectedNav=${(selectedNav) => {
+      setEventState({ ...eventState, selectedNav });
+    }}
+    collapsed=${eventState.collapsed}
+    onCollapsed=${(collapsed) => {
+      setEventState({ ...eventState, collapsed });
+    }}
+  >
     ${panels}
   </${EventPanel}>`;
 };

inspect_ai/_view/www/src/samples/transcript/InputEventView.mjs

@@ -12,11 +12,33 @@ import { formatDateTime } from "../../utils/Format.mjs";
  * @param { string  } props.id - The id of this event.
  * @param {import("../../types/log").InputEvent} props.event - The event object to display.
  * @param { Object } props.style - The style of this event.
+ * @param {import("./Types.mjs").TranscriptEventState} props.eventState - The state for this event
+ * @param {(state: import("./Types.mjs").TranscriptEventState) => void} props.setEventState - Update the state for this event
  * @returns {import("preact").JSX.Element} The component.
  */
-export const InputEventView = ({ id, event, style }) => {
+export const InputEventView = ({
+  id,
+  event,
+  style,
+  eventState,
+  setEventState,
+}) => {
   return html`
-  <${EventPanel} id=${id} title="Input" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.input} style=${style}>
+  <${EventPanel} 
+    id=${id} 
+    title="Input" 
+    subTitle=${formatDateTime(new Date(event.timestamp))} 
+    icon=${ApplicationIcons.input} 
+    style=${style}
+    selectedNav=${eventState.selectedNav || ""}
+    onSelectedNav=${(selectedNav) => {
+      setEventState({ ...eventState, selectedNav });
+    }}
+    collapsed=${eventState.collapsed}
+    onCollapsed=${(collapsed) => {
+      setEventState({ ...eventState, collapsed });
+    }}
+    >
     <${ANSIDisplay} output=${event.input_ansi} style=${{ fontSize: "clamp(0.4rem, 1.15vw, 0.9rem)", ...style }}/>
   </${EventPanel}>`;
 };