inspect-ai 0.3.93__py3-none-any.whl → 0.3.95__py3-none-any.whl

inspect_ai/_view/www/src/app/samples/sample-tools/sample-filter/SampleFilter.tsx

@@ -14,28 +14,18 @@ import {
 import { tags } from "@lezer/highlight";
 import clsx from "clsx";
 import { EditorView, minimalSetup } from "codemirror";
-import { FC, useEffect, useMemo, useRef, useState } from "react";
+import { FC, useCallback, useEffect, useMemo, useRef } from "react";
 
-import { ScoreFilter } from "../../../../app/types";
-import { SampleSummary } from "../../../../client/api/types";
 import { useEvalDescriptor } from "../../../../state/hooks";
-import { EvalDescriptor } from "../../descriptor/types";
-import { FilterError, filterSamples, scoreFilterItems } from "../filters";
+import { useStore } from "../../../../state/store";
+import { debounce } from "../../../../utils/sync";
+import { FilterError } from "../../../types";
+import { sampleFilterItems } from "../filters";
 import { getCompletions } from "./completions";
 import styles from "./SampleFilter.module.css";
 import { language } from "./tokenize";
 
-// Types
-interface FilteringResult {
-  numSamples: number;
-  error?: FilterError;
-}
-
-interface SampleFilterProps {
-  samples: SampleSummary[];
-  scoreFilter: ScoreFilter;
-  setScoreFilter: (filter: ScoreFilter) => void;
-}
+interface SampleFilterProps {}
 
 // Constants
 const FILTER_TOOLTIP = `
@@ -105,20 +95,6 @@ const editorTheme = EditorView.theme({
   },
 });
 
-// Helper functions
-const getFilteringResult = (
-  evalDescriptor: EvalDescriptor,
-  sampleSummaries: SampleSummary[],
-  filterValue: string,
-): FilteringResult => {
-  const { result, error } = filterSamples(
-    evalDescriptor,
-    sampleSummaries,
-    filterValue,
-  );
-  return { numSamples: result.length, error };
-};
-
 const ensureOneLine = (tr: Transaction): TransactionSpec => {
   const newDoc = tr.newDoc.toString();
   if (!newDoc.includes("\n")) return tr;
@@ -154,11 +130,7 @@ const getLints = (
 };
 
 // Main component
-export const SampleFilter: FC<SampleFilterProps> = ({
-  samples,
-  scoreFilter,
-  setScoreFilter,
-}) => {
+export const SampleFilter: FC<SampleFilterProps> = () => {
   const editorRef = useRef<HTMLDivElement>(null);
   const editorViewRef = useRef<EditorView>(null);
   const linterCompartment = useRef<Compartment>(new Compartment());
@@ -167,43 +139,51 @@ export const SampleFilter: FC<SampleFilterProps> = ({
   const evalDescriptor = useEvalDescriptor();
 
   const filterItems = useMemo(
-    () => (evalDescriptor ? scoreFilterItems(evalDescriptor) : []),
+    () => (evalDescriptor ? sampleFilterItems(evalDescriptor) : []),
     [evalDescriptor],
   );
 
-  const [filteringResultInstant, setFilteringResultInstant] =
-    useState<FilteringResult | null>(null);
+  const filter = useStore((state) => state.log.filter);
+  const filterError = useStore((state) => state.log.filterError);
+  const setFilter = useStore((state) => state.logActions.setFilter);
 
-  const handleFocus = (event: FocusEvent, view: EditorView) => {
+  const handleFocus = useCallback((event: FocusEvent, view: EditorView) => {
     if (event.isTrusted && view.state.doc.toString() === "") {
       setTimeout(() => startCompletion(view), 0);
     }
-  };
+  }, []);
 
-  const makeAutocompletion = () =>
-    autocompletion({
-      override: [(context) => getCompletions(context, filterItems)],
-      activateOnCompletion: (c) => c.label.endsWith(" "),
-    });
+  const makeAutocompletion = useCallback(
+    () =>
+      autocompletion({
+        override: [(context) => getCompletions(context, filterItems)],
+        activateOnCompletion: (c) => c.label.endsWith(" "),
+      }),
+    [],
+  );
 
-  const makeLinter = () =>
-    linter((view) => getLints(view, filteringResultInstant?.error));
+  const makeLinter = useCallback(
+    () => linter((view) => getLints(view, filterError)),
+    [filterError],
+  );
 
-  const makeUpdateListener = () =>
-    EditorView.updateListener.of((update) => {
-      if (update.docChanged && evalDescriptor) {
-        const newValue = update.state.doc.toString();
-        const filteringResult = getFilteringResult(
-          evalDescriptor,
-          samples,
-          newValue,
-        );
-        if (!filteringResult.error) {
-          setScoreFilter({ value: newValue });
+  const debounceSetFilter = useCallback(
+    debounce((value: string) => {
+      setFilter(value);
+    }, 200),
+    [setFilter],
+  );
+
+  const makeUpdateListener = useCallback(
+    () =>
+      EditorView.updateListener.of((update) => {
+        if (update.docChanged && evalDescriptor) {
+          const newValue = update.state.doc.toString();
+          debounceSetFilter(newValue);
         }
-        setFilteringResultInstant(filteringResult);
-      }
-    });
+      }),
+    [setFilter],
+  );
 
   // Initialize editor
   useEffect(() => {
@@ -212,7 +192,7 @@ export const SampleFilter: FC<SampleFilterProps> = ({
     editorViewRef.current = new EditorView({
       parent: editorRef.current ?? undefined,
       state: EditorState.create({
-        doc: scoreFilter.value || "",
+        doc: filter || "",
         extensions: [
           minimalSetup,
           bracketMatching(),
@@ -236,21 +216,16 @@ export const SampleFilter: FC<SampleFilterProps> = ({
     if (!editorViewRef.current) return;
 
     const currentValue = editorViewRef.current.state.doc.toString();
-    if (scoreFilter.value === currentValue) return;
+    if (filter === currentValue) return;
 
-    if (evalDescriptor) {
-      setFilteringResultInstant(
-        getFilteringResult(evalDescriptor, samples, scoreFilter.value || ""),
-      );
-    }
     editorViewRef.current.dispatch({
       changes: {
         from: 0,
         to: currentValue.length,
-        insert: scoreFilter.value || "",
+        insert: filter || "",
       },
     });
-  }, [evalDescriptor, scoreFilter.value]);
+  }, [filter]);
 
   // Update compartments when dependencies change
   useEffect(() => {
@@ -271,7 +246,7 @@ export const SampleFilter: FC<SampleFilterProps> = ({
     editorViewRef.current?.dispatch({
       effects: linterCompartment.current.reconfigure(makeLinter()),
     });
-  }, [filteringResultInstant?.error]);
+  }, [filterError]);
 
   return (
     <div style={{ display: "flex" }}>
@@ -288,10 +263,7 @@ export const SampleFilter: FC<SampleFilterProps> = ({
       </span>
       <div
         ref={editorRef}
-        className={clsx(
-          filteringResultInstant?.error && "filter-pending",
-          styles.input,
-        )}
+        className={clsx(filterError && "filter-pending", styles.input)}
       />
       <span
         className={clsx("bi", "bi-question-circle", styles.help)}

inspect_ai/_view/www/src/app/samples/sample-tools/sample-filter/completions.ts

@@ -12,7 +12,7 @@ import {
   kScoreTypeOther,
   kScoreTypePassFail,
 } from "../../../../constants";
-import { ScoreFilterItem } from "../filters";
+import { SampleFilterItem } from "../filters";
 import {
   KEYWORDS,
   MATH_FUNCTIONS,
@@ -29,7 +29,7 @@ interface CompletionOptions {
 }
 
 interface CanonicalNameCompletionProps {
-  autoSpaceIf?: (item: ScoreFilterItem) => boolean;
+  autoSpaceIf?: (item: SampleFilterItem) => boolean;
 }
 
 const isLiteral = (token: Token): boolean =>
@@ -98,7 +98,7 @@ const makeLiteralCompletion = (k: string): Completion => ({
 });
 
 const makeCanonicalNameCompletion = (
-  item: ScoreFilterItem,
+  item: SampleFilterItem,
   { autoSpaceIf = () => false }: CanonicalNameCompletionProps = {},
 ): Completion => ({
   label: item.canonicalName + (autoSpaceIf(item) ? " " : ""),
@@ -107,7 +107,7 @@ const makeCanonicalNameCompletion = (
   boost: 30,
 });
 
-const makeMemberAccessCompletion = (item: ScoreFilterItem): Completion => ({
+const makeMemberAccessCompletion = (item: SampleFilterItem): Completion => ({
   label: item.qualifiedName?.split(".")[1] || "",
   type: "variable",
   info: item.tooltip,
@@ -115,9 +115,9 @@ const makeMemberAccessCompletion = (item: ScoreFilterItem): Completion => ({
 });
 
 const getMemberScoreItems = (
-  filterItems: ScoreFilterItem[],
+  filterItems: SampleFilterItem[],
   scorer: string,
-): ScoreFilterItem[] =>
+): SampleFilterItem[] =>
   filterItems.filter((item) => item?.qualifiedName?.startsWith(`${scorer}.`));
 
 /**
@@ -136,7 +136,7 @@ const getMemberScoreItems = (
  */
 export function getCompletions(
   context: CompletionContext,
-  filterItems: ScoreFilterItem[],
+  filterItems: SampleFilterItem[],
 ): CompletionResult | null {
   const keywordCompletionItems = KEYWORDS.map(makeKeywordCompletion);
   const mathFunctionCompletionItems = MATH_FUNCTIONS.map(
@@ -177,7 +177,7 @@ export function getCompletions(
   const completionStart = currentToken ? currentToken.from : context.pos;
   const completingAtEnd = context.pos === doc.length;
 
-  const findFilterItem = (endIndex: number): ScoreFilterItem | undefined => {
+  const findFilterItem = (endIndex: number): SampleFilterItem | undefined => {
     if (prevToken(endIndex)?.type !== "variable") return undefined;
 
     let name = prevToken(endIndex).text;
@@ -267,7 +267,7 @@ export function getCompletions(
 
   const variableCompletions = () => makeCompletions(variableCompletionItems);
 
-  const memberAccessCompletions = (items: ScoreFilterItem[]) =>
+  const memberAccessCompletions = (items: SampleFilterItem[]) =>
     makeCompletions(items.map(makeMemberAccessCompletion), {
       autocompleteInTheMiddle: true,
       includeDefault: false,

inspect_ai/_view/www/src/app/samples/transcript/SandboxEventView.module.css

@@ -13,7 +13,7 @@
 }
 
 .exec {
-  margin-top: 0.5em;
+  margin-top: 0;
 }
 
 .result {
@@ -29,4 +29,5 @@
   white-space: pre-wrap;
   word-wrap: break-word;
   overflow-wrap: break-word;
+  margin-bottom: 0;
 }

inspect_ai/_view/www/src/app/samples/transcript/SpanEventView.tsx

@@ -0,0 +1,174 @@
+import clsx from "clsx";
+import { FC } from "react";
+import { SpanBeginEvent } from "../../../@types/log";
+import { formatDateTime } from "../../../utils/format";
+import { EventPanel } from "./event/EventPanel";
+import { TranscriptComponent } from "./TranscriptView";
+import { kSandboxSignalName } from "./transform/fixups";
+import { EventNode } from "./types";
+
+interface SpanEventViewProps {
+  id: string;
+  event: SpanBeginEvent;
+  children: EventNode[];
+  className?: string | string[];
+}
+
+/**
+ * Renders the SpanEventView component.
+ */
+export const SpanEventView: FC<SpanEventViewProps> = ({
+  id,
+  event,
+  children,
+  className,
+}) => {
+  const descriptor = spanDescriptor(event);
+  const title =
+    descriptor.name ||
+    `${event.type ? event.type + ": " : "Step: "}${event.name}`;
+  const text = summarize(children);
+
+  return (
+    <EventPanel
+      id={`span-${event.name}-${id}`}
+      className={clsx("transcript-span", className)}
+      title={title}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      text={text}
+      collapse={descriptor.collapse}
+      icon={descriptor.icon}
+    >
+      <TranscriptComponent
+        id={`span|${event.name}|${id}`}
+        eventNodes={children}
+      />
+    </EventPanel>
+  );
+};
+
+const summarize = (children: EventNode[]) => {
+  if (children.length === 0) {
+    return "(no events)";
+  }
+
+  const formatEvent = (event: string, count: number) => {
+    if (count === 1) {
+      return `${count} ${event} event`;
+    } else {
+      return `${count} ${event} events`;
+    }
+  };
+
+  // Count the types
+  const typeCount: Record<string, number> = {};
+  children.forEach((child) => {
+    const currentCount = typeCount[child.event.event] || 0;
+    typeCount[child.event.event] = currentCount + 1;
+  });
+
+  // Try to summarize event types
+  const numberOfTypes = Object.keys(typeCount).length;
+  if (numberOfTypes < 3) {
+    return Object.keys(typeCount)
+      .map((key) => {
+        return formatEvent(key, typeCount[key]);
+      })
+      .join(", ");
+  }
+
+  // To many types, just return the number of events
+  if (children.length === 1) {
+    return "1 event";
+  } else {
+    return `${children.length} events`;
+  }
+};
+
+/**
+ * Returns a descriptor object containing icon and style based on the event type and name.
+ */
+const spanDescriptor = (
+  event: SpanBeginEvent,
+): { icon?: string; name?: string; endSpace?: boolean; collapse?: boolean } => {
+  const rootStepDescriptor = {
+    endSpace: true,
+  };
+
+  if (event.type === "solver") {
+    switch (event.name) {
+      case "chain_of_thought":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "generate":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "self_critique":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "system_message":
+        return {
+          ...rootStepDescriptor,
+          collapse: true,
+        };
+      case "use_tools":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "multiple_choice":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      default:
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+    }
+  } else if (event.type === "scorer") {
+    return {
+      ...rootStepDescriptor,
+      collapse: false,
+    };
+  } else if (event.event === "span_begin") {
+    if (event.span_id === kSandboxSignalName) {
+      return {
+        ...rootStepDescriptor,
+        name: "Sandbox Events",
+        collapse: true,
+      };
+    } else if (event.name === "init") {
+      return {
+        ...rootStepDescriptor,
+        name: "Init",
+        collapse: true,
+      };
+    } else {
+      return {
+        ...rootStepDescriptor,
+        collapse: false,
+      };
+    }
+  } else {
+    switch (event.name) {
+      case "sample_init":
+        return {
+          ...rootStepDescriptor,
+          name: "Sample Init",
+          collapse: true,
+        };
+      default:
+        return {
+          endSpace: false,
+        };
+    }
+  }
+};

inspect_ai/_view/www/src/app/samples/transcript/ToolEventView.tsx

@@ -4,7 +4,7 @@ import { resolveToolInput } from "../chat/tools/tool";
 import { ToolCallView } from "../chat/tools/ToolCallView";
 import { ApprovalEventView } from "./ApprovalEventView";
 import { EventPanel } from "./event/EventPanel";
-import { TranscriptView } from "./TranscriptView";
+import { TranscriptComponent } from "./TranscriptView";
 
 import clsx from "clsx";
 import { FC, useMemo } from "react";
@@ -12,11 +12,12 @@ import { PulsingDots } from "../../../components/PulsingDots";
 import { ChatView } from "../chat/ChatView";
 import { formatTiming, formatTitle } from "./event/utils";
 import styles from "./ToolEventView.module.css";
+import { EventNode } from "./types";
 
 interface ToolEventViewProps {
   id: string;
   event: ToolEvent;
-  depth: number;
+  children: EventNode[];
   className?: string | string[];
 }
 
@@ -26,7 +27,7 @@ interface ToolEventViewProps {
 export const ToolEventView: FC<ToolEventViewProps> = ({
   id,
   event,
-  depth,
+  children,
   className,
 }) => {
   // Extract tool input
@@ -92,13 +93,12 @@ export const ToolEventView: FC<ToolEventViewProps> = ({
           </div>
         ) : undefined}
       </div>
-      {event.events.length > 0 ? (
-        <TranscriptView
-          id={`${id}-subtask`}
+      {children.length > 0 ? (
+        <TranscriptComponent
           data-name="Transcript"
+          id={`${id}-subtask`}
+          eventNodes={children}
           data-default={event.failed || event.agent ? true : null}
-          events={event.events}
-          depth={depth + 1}
         />
       ) : (
         ""

inspect_ai/_view/www/src/app/samples/transcript/TranscriptView.tsx

@@ -17,6 +17,7 @@ import { ToolEventView } from "./ToolEventView";
 import { EventNode } from "./types";
 
 import clsx from "clsx";
+import { SpanEventView } from "./SpanEventView";
 import styles from "./TranscriptView.module.css";
 import { TranscriptVirtualListComponent } from "./TranscriptVirtualListComponent";
 import { fixupEventStream } from "./transform/fixups";
@@ -64,7 +65,6 @@ export const TranscriptVirtualList: FC<TranscriptVirtualListProps> = memo(
     const eventNodes = useMemo(() => {
       const resolvedEvents = fixupEventStream(events, !running);
       const eventNodes = treeifyEvents(resolvedEvents, depth || 0);
-
       return eventNodes;
     }, [events, depth]);
 
@@ -201,6 +201,16 @@ export const RenderedEventNode: FC<RenderedEventNodeProps> = memo(
           <StateEventView id={id} event={node.event} className={className} />
         );
 
+      case "span_begin":
+        return (
+          <SpanEventView
+            id={id}
+            event={node.event}
+            children={node.children}
+            className={className}
+          />
+        );
+
       case "step":
         return (
           <StepEventView
@@ -237,7 +247,7 @@ export const RenderedEventNode: FC<RenderedEventNodeProps> = memo(
             id={id}
             event={node.event}
             className={className}
-            depth={node.depth}
+            children={node.children}
           />
         );
 

inspect_ai/_view/www/src/app/samples/transcript/TranscriptVirtualListComponent.module.css

@@ -8,7 +8,7 @@
 
 .node {
   padding-top: 0.7rem;
-  padding-bottom: 0em;
+  padding-bottom: 1px;
 }
 
 .attached {

inspect_ai/_view/www/src/app/samples/transcript/event/EventPanel.tsx

@@ -9,7 +9,6 @@ import {
 import { ApplicationIcons } from "../../../appearance/icons";
 import { EventNavs } from "./EventNavs";
 
-import { ProgressBar } from "../../../../components/ProgressBar";
 import { useProperty } from "../../../../state/hooks";
 import styles from "./EventPanel.module.css";
 
@@ -41,7 +40,6 @@ export const EventPanel: FC<EventPanelProps> = ({
   icon,
   collapse,
   children,
-  running,
 }) => {
   const [isCollapsed, setCollapsed] = useProperty(id, "collapsed", {
     defaultValue: !!collapse,
@@ -191,7 +189,6 @@ export const EventPanel: FC<EventPanelProps> = ({
           })}
         </div>
       </div>
-      <ProgressBar animating={!!running} />
     </>
   );
   return card;