inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/samples/transcript/event/EventNav.tsx

@@ -1,6 +1,6 @@
 import clsx from "clsx";
 
-import { FC } from "react";
+import { FC, useCallback } from "react";
 import styles from "./EventNav.module.css";
 
 interface EventNavProps {
@@ -19,6 +19,11 @@ export const EventNav: FC<EventNavProps> = ({
   setSelectedNav,
 }) => {
   const active = target === selectedNav;
+
+  const handleClick = useCallback(() => {
+    setSelectedNav(target);
+  }, [setSelectedNav, target]);
+
   return (
     <li className="nav-item">
       <button
@@ -33,9 +38,7 @@ export const EventNav: FC<EventNavProps> = ({
           "text-size-small",
           styles.tab,
         )}
-        onClick={() => {
-          setSelectedNav(target);
-        }}
+        onClick={handleClick}
       >
         {title}
       </button>

inspect_ai/_view/www/src/samples/transcript/event/EventPanel.tsx

@@ -1,9 +1,16 @@
 import clsx from "clsx";
-import { ReactNode } from "react";
+import {
+  FC,
+  isValidElement,
+  ReactElement,
+  ReactNode,
+  useCallback,
+} from "react";
 import { ApplicationIcons } from "../../../appearance/icons";
 import { EventNavs } from "./EventNavs";
 
-import React from "react";
+import { ProgressBar } from "../../../components/ProgressBar";
+import { useProperty } from "../../../state/hooks";
 import styles from "./EventPanel.module.css";
 
 interface EventPanelProps {
@@ -14,11 +21,8 @@ interface EventPanelProps {
   text?: string;
   icon?: string;
   collapse?: boolean;
-  collapsed?: boolean;
-  setCollapsed: (collapse: boolean) => void;
-  selectedNav: string;
-  setSelectedNav: (nav: string) => void;
   children?: ReactNode | ReactNode[];
+  running?: boolean;
 }
 
 interface ChildProps {
@@ -28,7 +32,7 @@ interface ChildProps {
 /**
  * Renders the StateEventView component.
  */
-export const EventPanel: React.FC<EventPanelProps> = ({
+export const EventPanel: FC<EventPanelProps> = ({
   id,
   className,
   title,
@@ -36,23 +40,30 @@ export const EventPanel: React.FC<EventPanelProps> = ({
   text,
   icon,
   collapse,
-  collapsed,
-  setCollapsed,
   children,
-  setSelectedNav,
-  selectedNav,
+  running,
 }) => {
+  const [isCollapsed, setCollapsed] = useProperty(id, "collapsed", {
+    defaultValue: !!collapse,
+  });
+
   const hasCollapse = collapse !== undefined;
-  const isCollapsed = collapsed === undefined ? collapse : collapsed;
 
   const pillId = (index: number) => {
     return `${id}-nav-pill-${index}`;
   };
-
   const filteredArrChildren = (
     Array.isArray(children) ? children : [children]
   ).filter((child) => !!child);
-  const defaultPillId = pillId(0);
+
+  const defaultPill = filteredArrChildren.findIndex((node) => {
+    return hasDataDefault(node) && node.props["data-default"];
+  });
+  const defaultPillId = defaultPill !== -1 ? pillId(defaultPill) : pillId(0);
+
+  const [selectedNav, setSelectedNav] = useProperty(id, "selectedNav", {
+    defaultValue: defaultPillId,
+  });
 
   const gridColumns = [];
 
@@ -72,6 +83,10 @@ export const EventPanel: React.FC<EventPanelProps> = ({
   gridColumns.push("minmax(0, max-content)");
   gridColumns.push("minmax(0, max-content)");
 
+  const toggleCollapse = useCallback(() => {
+    setCollapsed(!isCollapsed);
+  }, [setCollapsed, isCollapsed]);
+
   const titleEl =
     title || icon || filteredArrChildren.length > 1 ? (
       <div
@@ -86,9 +101,7 @@ export const EventPanel: React.FC<EventPanelProps> = ({
       >
         {hasCollapse ? (
           <i
-            onClick={() => {
-              setCollapsed(!isCollapsed);
-            }}
+            onClick={toggleCollapse}
             className={
               isCollapsed
                 ? ApplicationIcons.chevron.right
@@ -104,33 +117,23 @@ export const EventPanel: React.FC<EventPanelProps> = ({
               icon || ApplicationIcons.metadata,
               "text-style-secondary",
             )}
-            onClick={() => {
-              setCollapsed(!isCollapsed);
-            }}
+            onClick={toggleCollapse}
           />
         ) : (
           ""
         )}
         <div
           className={clsx("text-style-secondary", "text-style-label")}
-          onClick={() => {
-            setCollapsed(!isCollapsed);
-          }}
+          onClick={toggleCollapse}
         >
           {title}
         </div>
-        <div
-          onClick={() => {
-            setCollapsed(!isCollapsed);
-          }}
-        ></div>
+        <div onClick={toggleCollapse}></div>
         <div
           className={clsx("text-style-secondary", styles.label)}
-          onClick={() => {
-            setCollapsed(!isCollapsed);
-          }}
+          onClick={toggleCollapse}
         >
-          {collapsed ? text : ""}
+          {isCollapsed ? text : ""}
         </div>
         <div className={styles.navs}>
           {(!hasCollapse || !isCollapsed) &&
@@ -140,7 +143,7 @@ export const EventPanel: React.FC<EventPanelProps> = ({
               navs={filteredArrChildren.map((child, index) => {
                 const defaultTitle = `Tab ${index}`;
                 const title =
-                  child && React.isValidElement<ChildProps>(child)
+                  child && isValidElement<ChildProps>(child)
                     ? (child.props as ChildProps)["data-name"] || defaultTitle
                     : defaultTitle;
                 return {
@@ -149,7 +152,7 @@ export const EventPanel: React.FC<EventPanelProps> = ({
                   target: pillId(index),
                 };
               })}
-              selectedNav={selectedNav || defaultPillId}
+              selectedNav={selectedNav}
               setSelectedNav={setSelectedNav}
             />
           ) : (
@@ -162,33 +165,51 @@ export const EventPanel: React.FC<EventPanelProps> = ({
     );
 
   const card = (
-    <div id={id} className={clsx(className, styles.card)}>
-      {titleEl}
-      <div
-        className={clsx(
-          "tab-content",
-          styles.cardContent,
-          hasCollapse && isCollapsed ? styles.hidden : undefined,
-        )}
-      >
-        {filteredArrChildren?.map((child, index) => {
-          const id = pillId(index);
-          const isSelected = selectedNav
-            ? id === selectedNav
-            : id === defaultPillId;
-
-          return (
-            <div
-              key={`children-${id}-${index}`}
-              id={id}
-              className={clsx("tab-pane", "show", isSelected ? "active" : "")}
-            >
-              {child}
-            </div>
-          );
-        })}
+    <>
+      <div id={id} className={clsx(className, styles.card)}>
+        {titleEl}
+        <div
+          className={clsx(
+            "tab-content",
+            styles.cardContent,
+            hasCollapse && isCollapsed ? styles.hidden : undefined,
+          )}
+        >
+          {filteredArrChildren?.map((child, index) => {
+            const id = pillId(index);
+            const isSelected = id === selectedNav;
+
+            return (
+              <div
+                key={`children-${id}-${index}`}
+                id={id}
+                className={clsx("tab-pane", "show", isSelected ? "active" : "")}
+              >
+                {child}
+              </div>
+            );
+          })}
+        </div>
       </div>
-    </div>
+      <ProgressBar animating={!!running} />
+    </>
   );
   return card;
 };
+
+// Typeguard for reading default value from pills
+interface DataDefaultProps {
+  "data-default"?: boolean;
+  [key: string]: any;
+}
+
+function hasDataDefault(
+  node: ReactNode,
+): node is ReactElement<DataDefaultProps> {
+  return (
+    isValidElement(node) &&
+    node.props !== null &&
+    typeof node.props === "object" &&
+    "data-default" in node.props
+  );
+}

inspect_ai/_view/www/src/samples/transcript/event/EventProgressPanel.module.css

@@ -0,0 +1,23 @@
+.panel {
+  display: flex;
+  justify-content: center;
+  padding-top: 1em;
+  padding-bottom: 1em;
+}
+
+.container {
+  display: grid;
+  grid-template-columns: max-content max-content;
+  column-gap: 0.3em;
+}
+
+.spinner {
+  height: 15px;
+  width: 14px;
+  border-width: 2px;
+  color: var(--bs-secondary);
+}
+
+.text {
+  margin-top: -2px;
+}

inspect_ai/_view/www/src/samples/transcript/event/EventProgressPanel.tsx

@@ -0,0 +1,27 @@
+import clsx from "clsx";
+import { FC } from "react";
+
+import styles from "./EventProgressPanel.module.css";
+
+interface EventProgressPanelProps {
+  text: string;
+}
+
+export const EventProgressPanel: FC<EventProgressPanelProps> = ({ text }) => {
+  return (
+    <div className={clsx(styles.panel)}>
+      <div className={clsx(styles.container)}>
+        <Spinner />
+        <div className={clsx("text-size-smaller", styles.text)}>{text}</div>
+      </div>
+    </div>
+  );
+};
+
+const Spinner: FC = () => {
+  return (
+    <div className={clsx(styles.spinner, "spinner-border")} role="status">
+      <span className={clsx("visually-hidden")}>generating...</span>
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.tsx

@@ -17,7 +17,8 @@ interface Signature {
 
 interface ChangeType {
   type: string;
-  signature: Signature;
+  signature?: Signature;
+  match?: (changes: JsonChange[]) => boolean;
   render: (
     changes: JsonChange[],
     state: Record<string, unknown>,
@@ -211,8 +212,35 @@ const renderTools = (
   );
 };
 
+const createMessageRenderer = (name: string, role: string): ChangeType => {
+  return {
+    type: name,
+    match: (changes: JsonChange[]) => {
+      if (changes.length === 1) {
+        const change = changes[0];
+        if (change.op === "add" && change.path.match(/\/messages\/\d+/)) {
+          return change.value["role"] === role;
+        }
+      }
+      return false;
+    },
+    render: (changes) => {
+      const message = changes[0].value as unknown;
+      return (
+        <ChatView
+          key="system_msg_event_preview"
+          id="system_msg_event_preview"
+          messages={[message] as Messages}
+        />
+      );
+    },
+  };
+};
+
 export const RenderableChangeTypes: ChangeType[] = [
   system_msg_added_sig,
+  createMessageRenderer("assistant_msg", "assistant"),
+  createMessageRenderer("user_msg", "user"),
   use_tools,
   add_tools,
 ];

inspect_ai/_view/www/src/samples/transcript/state/StateEventView.tsx

@@ -10,7 +10,6 @@ import {
 } from "../../../types/log";
 import { formatDateTime } from "../../../utils/format";
 import { EventPanel } from "../event/EventPanel";
-import { TranscriptEventState } from "../types";
 import { StateDiffView } from "./StateDiffView";
 import {
   RenderableChangeTypes,
@@ -23,8 +22,6 @@ import styles from "./StateEventView.module.css";
 interface StateEventViewProps {
   id: string;
   event: StateEvent | StoreEvent;
-  eventState: TranscriptEventState;
-  setEventState: (state: TranscriptEventState) => void;
   isStore?: boolean;
   className?: string | string[];
 }
@@ -35,8 +32,6 @@ interface StateEventViewProps {
 export const StateEventView: FC<StateEventViewProps> = ({
   id,
   event,
-  eventState,
-  setEventState,
   isStore = false,
   className,
 }) => {
@@ -46,7 +41,15 @@ export const StateEventView: FC<StateEventViewProps> = ({
 
   // Synthesize objects for comparison
   const [before, after] = useMemo(() => {
-    return synthesizeComparable(event.changes);
+    try {
+      return synthesizeComparable(event.changes);
+    } catch (e) {
+      console.error(
+        "Unable to synthesize comparable object to display state diffs.",
+        e,
+      );
+      return [{}, {}];
+    }
   }, [event.changes]);
 
   // This clone is important since the state is used by react as potential values that are rendered
@@ -66,14 +69,6 @@ export const StateEventView: FC<StateEventViewProps> = ({
       subTitle={formatDateTime(new Date(event.timestamp))}
       text={!changePreview ? summary : undefined}
       collapse={changePreview === undefined ? true : undefined}
-      selectedNav={eventState.selectedNav || ""}
-      setSelectedNav={(selectedNav) => {
-        setEventState({ ...eventState, selectedNav });
-      }}
-      collapsed={eventState.collapsed}
-      setCollapsed={(collapsed) => {
-        setEventState({ ...eventState, collapsed });
-      }}
     >
       {changePreview ? (
         <div data-name="Summary" className={clsx(styles.summary)}>
@@ -103,59 +98,71 @@ const generatePreview = (
     ...RenderableChangeTypes,
     ...(isStore ? StoreSpecificRenderableTypes : []),
   ]) {
-    // Note that we currently only have renderers that depend upon
-    // add, remove, replace, but we should likely add
-    // move, copy, test
-    const requiredMatchCount =
-      changeType.signature.remove.length +
-      changeType.signature.replace.length +
-      changeType.signature.add.length;
-    let matchingOps = 0;
-    for (const change of changes) {
-      const op = change.op;
-      switch (op) {
-        case "add":
-          if (changeType.signature.add && changeType.signature.add.length > 0) {
-            changeType.signature.add.forEach((signature) => {
-              if (change.path.match(signature)) {
-                matchingOps++;
-              }
-            });
-          }
-          break;
-        case "remove":
-          if (
-            changeType.signature.remove &&
-            changeType.signature.remove.length > 0
-          ) {
-            changeType.signature.remove.forEach((signature) => {
-              if (change.path.match(signature)) {
-                matchingOps++;
-              }
-            });
-          }
-          break;
-        case "replace":
-          if (
-            changeType.signature.replace &&
-            changeType.signature.replace.length > 0
-          ) {
-            changeType.signature.replace.forEach((signature) => {
-              if (change.path.match(signature)) {
-                matchingOps++;
-              }
-            });
-          }
-          break;
+    if (changeType.signature) {
+      // Note that we currently only have renderers that depend upon
+      // add, remove, replace, but we should likely add
+      // move, copy, test
+      const requiredMatchCount =
+        changeType.signature.remove.length +
+        changeType.signature.replace.length +
+        changeType.signature.add.length;
+      let matchingOps = 0;
+      for (const change of changes) {
+        const op = change.op;
+        switch (op) {
+          case "add":
+            if (
+              changeType.signature.add &&
+              changeType.signature.add.length > 0
+            ) {
+              changeType.signature.add.forEach((signature) => {
+                if (change.path.match(signature)) {
+                  matchingOps++;
+                }
+              });
+            }
+            break;
+          case "remove":
+            if (
+              changeType.signature.remove &&
+              changeType.signature.remove.length > 0
+            ) {
+              changeType.signature.remove.forEach((signature) => {
+                if (change.path.match(signature)) {
+                  matchingOps++;
+                }
+              });
+            }
+            break;
+          case "replace":
+            if (
+              changeType.signature.replace &&
+              changeType.signature.replace.length > 0
+            ) {
+              changeType.signature.replace.forEach((signature) => {
+                if (change.path.match(signature)) {
+                  matchingOps++;
+                }
+              });
+            }
+            break;
+        }
+      }
+      if (matchingOps === requiredMatchCount) {
+        const el = changeType.render(changes, resolvedState);
+        results.push(el);
+        // Only one renderer can process a change
+        // TODO: consider changing this to allow many handlers to render (though then we sort of need
+        // to match the renderer to the key (e.g. a rendered for `tool_choice` a renderer for `tools` etc..))
+        break;
+      }
+    } else if (changeType.match) {
+      const matches = changeType.match(changes);
+      if (matches) {
+        const el = changeType.render(changes, resolvedState);
+        results.push(el);
+        break;
       }
-    }
-    if (matchingOps === requiredMatchCount) {
-      const el = changeType.render(changes, resolvedState);
-      results.push(el);
-      // Only one renderer can process a change
-      // TODO: consider changing this to allow many handlers to render (though then we sort of need
-      // to match the renderer to the key (e.g. a rendered for `tool_choice` a renderer for `tools` etc..))
-      break;
     }
   }
   return results.length > 0 ? results : undefined;
@@ -269,21 +276,44 @@ function setPath(
   value: unknown,
 ): void {
   const keys = parsePath(path);
-  let current: Record<string, unknown> = target;
+  let current: Record<string, unknown> | unknown[] = target;
 
   for (let i = 0; i < keys.length - 1; i++) {
     const key = keys[i];
-    if (!(key in current)) {
-      // If the next key is a number, create an array, otherwise an object
-      current[key] = isArrayIndex(keys[i + 1]) ? [] : {};
+
+    if (Array.isArray(current)) {
+      const numericIndex = getIndex(key);
+      current[numericIndex] = isArrayIndex(keys[i + 1]) ? [] : {};
+      current = current[numericIndex] as
+        | Record<string, unknown>
+        | Array<unknown>;
+    } else {
+      if (!(key in current)) {
+        // If the next key is a number, create an array, otherwise an object
+        current[key] = isArrayIndex(keys[i + 1]) ? [] : {};
+      }
+      current = current[key] as Record<string, unknown> | Array<unknown>;
     }
-    current = current[key] as Record<string, unknown>;
   }
 
   const lastKey = keys[keys.length - 1];
-  current[lastKey] = value;
+  if (Array.isArray(current)) {
+    const numericIndex = getIndex(lastKey);
+    current[numericIndex] = value;
+  } else {
+    current[lastKey] = value;
+  }
 }
 
+const getIndex = (key: string): number => {
+  const numericIndex = isArrayIndex(key) ? parseInt(key) : undefined;
+  if (numericIndex === undefined) {
+    throw new Error(`The key ${key} isn't a valid Array index!`);
+  }
+
+  return numericIndex;
+};
+
 /**
  * Places structure in an object (without placing values)
  */

inspect_ai/_view/www/src/scoring/utils.ts

@@ -0,0 +1,87 @@
+import { EvalSummary, SampleSummary } from "../api/types";
+import { EvalResults } from "../types/log";
+
+export interface ScorerInfo {
+  name: string;
+  scorer: string;
+}
+
+/**
+ * Extracts scorer information from evaluation results
+ */
+const getScorersFromResults = (results?: EvalResults): ScorerInfo[] => {
+  if (!results?.scores) {
+    return [];
+  }
+
+  return results.scores.reduce((uniqueScorers, score) => {
+    const isDuplicate = uniqueScorers.some(
+      (existing) =>
+        existing.scorer === score.scorer && existing.name === score.name,
+    );
+
+    if (!isDuplicate) {
+      uniqueScorers.push({
+        name: score.name,
+        scorer: score.scorer,
+      });
+    }
+
+    return uniqueScorers;
+  }, [] as ScorerInfo[]);
+};
+
+/**
+ * Extracts scorer information from sample summaries
+ */
+const getScorersFromSamples = (samples: SampleSummary[]): ScorerInfo[] => {
+  // Find a sample with scores
+  const scoredSample = samples.find((sample) => {
+    return !!sample.scores;
+  });
+
+  return Object.keys(scoredSample?.scores || {}).map((key) => ({
+    name: key,
+    scorer: key,
+  }));
+};
+
+/**
+ * Gets all available scorers for a log, prioritizing results over samples
+ */
+export const getAvailableScorers = (
+  log: EvalSummary,
+  sampleSummaries: SampleSummary[],
+): ScorerInfo[] | undefined => {
+  const resultScorers = log.results ? getScorersFromResults(log.results) : [];
+  if (resultScorers.length > 0) {
+    return resultScorers;
+  }
+
+  const sampleScorers = getScorersFromSamples(sampleSummaries);
+  if (sampleScorers.length > 0) {
+    return sampleScorers;
+  }
+
+  return undefined;
+};
+
+/**
+ * Gets the default scorer to use, preferring the first scorer from results
+ * or falling back to the first scorer from samples
+ */
+export const getDefaultScorer = (
+  log: EvalSummary,
+  sampleSummaries: SampleSummary[],
+): ScorerInfo | undefined => {
+  if (sampleSummaries.length === 0) {
+    return undefined;
+  }
+
+  const allScorers = getAvailableScorers(log, sampleSummaries);
+  if (allScorers) {
+    return allScorers[0];
+  } else {
+    return undefined;
+  }
+};