inspect-ai 0.3.93__py3-none-any.whl → 0.3.95__py3-none-any.whl

inspect_ai/_view/www/src/@types/log.d.ts

@@ -316,6 +316,7 @@ export type Explanation = string | null;
 export type Metadata6 = {
   [k: string]: unknown;
 } | null;
+export type SpanId = string | null;
 export type Timestamp = string;
 export type WorkingStart = number;
 export type Pending = boolean | null;
@@ -339,6 +340,7 @@ export type Files1 = {
 } | null;
 export type Setup1 = string | null;
 export type JsonValue = unknown;
+export type SpanId1 = string | null;
 export type Timestamp1 = string;
 export type WorkingStart1 = number;
 export type Pending1 = boolean | null;
@@ -352,6 +354,7 @@ export type Type10 =
   | "custom";
 export type Message2 = string;
 export type Limit1 = number | null;
+export type SpanId2 = string | null;
 export type Timestamp2 = string;
 export type WorkingStart2 = number;
 export type Pending2 = boolean | null;
@@ -366,6 +369,7 @@ export type Input2 = string | null;
 export type Result = number | null;
 export type Output = string | null;
 export type Completed = string | null;
+export type SpanId3 = string | null;
 export type Timestamp3 = string;
 export type WorkingStart3 = number;
 export type Pending3 = boolean | null;
@@ -374,11 +378,13 @@ export type Op = "remove" | "add" | "replace" | "move" | "test" | "copy";
 export type Path = string;
 export type From = string | null;
 export type Changes = JsonChange[];
+export type SpanId4 = string | null;
 export type Timestamp4 = string;
 export type WorkingStart4 = number;
 export type Pending4 = boolean | null;
 export type Event4 = "store";
 export type Changes1 = JsonChange[];
+export type SpanId5 = string | null;
 export type Timestamp5 = string;
 export type WorkingStart5 = number;
 export type Pending5 = boolean | null;
@@ -399,11 +405,13 @@ export type Additionalproperties1 = boolean;
 export type Tools1 = ToolInfo[];
 export type ToolChoice = ("auto" | "any" | "none") | ToolFunction;
 export type Name9 = string;
+export type Retries = number | null;
 export type Error1 = string | null;
 export type Cache = ("read" | "write") | null;
 export type Time1 = number | null;
 export type Completed1 = string | null;
 export type WorkingTime = number | null;
+export type SpanId6 = string | null;
 export type Timestamp6 = string;
 export type WorkingStart6 = number;
 export type Pending6 = boolean | null;
@@ -428,6 +436,7 @@ export type Result1 =
       | ContentVideo
     )[];
 export type Truncated = [unknown, unknown] | null;
+export type SpanId7 = string | null;
 export type Timestamp7 = string;
 export type WorkingStart7 = number;
 export type Pending7 = boolean | null;
@@ -441,22 +450,26 @@ export type Decision =
   | "escalate"
   | "terminate";
 export type Explanation1 = string | null;
+export type SpanId8 = string | null;
 export type Timestamp8 = string;
 export type WorkingStart8 = number;
 export type Pending8 = boolean | null;
 export type Event8 = "input";
 export type Input4 = string;
 export type InputAnsi = string;
+export type SpanId9 = string | null;
 export type Timestamp9 = string;
 export type WorkingStart9 = number;
 export type Pending9 = boolean | null;
 export type Event9 = "score";
 export type Target2 = string | string[] | null;
 export type Intermediate = boolean;
+export type SpanId10 = string | null;
 export type Timestamp10 = string;
 export type WorkingStart10 = number;
 export type Pending10 = boolean | null;
 export type Event10 = "error";
+export type SpanId11 = string | null;
 export type Timestamp11 = string;
 export type WorkingStart11 = number;
 export type Pending11 = boolean | null;
@@ -476,24 +489,42 @@ export type Created1 = number;
 export type Filename = string;
 export type Module = string;
 export type Lineno = number;
+export type SpanId12 = string | null;
 export type Timestamp12 = string;
 export type WorkingStart12 = number;
 export type Pending12 = boolean | null;
 export type Event12 = "info";
 export type Source4 = string | null;
+export type SpanId13 = string | null;
 export type Timestamp13 = string;
 export type WorkingStart13 = number;
 export type Pending13 = boolean | null;
-export type Event13 = "step";
-export type Action1 = "begin" | "end";
+export type Event13 = "span_begin";
+export type Id8 = string;
+export type ParentId = string | null;
 export type Type13 = string | null;
 export type Name11 = string;
+export type SpanId14 = string | null;
 export type Timestamp14 = string;
 export type WorkingStart14 = number;
 export type Pending14 = boolean | null;
-export type Event14 = "subtask";
-export type Name12 = string;
+export type Event14 = "span_end";
+export type Id9 = string;
+export type SpanId15 = string | null;
+export type Timestamp15 = string;
+export type WorkingStart15 = number;
+export type Pending15 = boolean | null;
+export type Event15 = "step";
+export type Action1 = "begin" | "end";
 export type Type14 = string | null;
+export type Name12 = string;
+export type SpanId16 = string | null;
+export type Timestamp16 = string;
+export type WorkingStart16 = number;
+export type Pending16 = boolean | null;
+export type Event16 = "subtask";
+export type Name13 = string;
+export type Type15 = string | null;
 export type Events2 = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -508,6 +539,8 @@ export type Events2 = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -527,6 +560,8 @@ export type Events1 = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -548,6 +583,8 @@ export type Events = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -555,7 +592,7 @@ export type TotalTime = number | null;
 export type WorkingTime3 = number | null;
 export type Uuid = string | null;
 export type ErrorRetries = EvalError[] | null;
-export type Type15 =
+export type Type16 =
   | "context"
   | "time"
   | "working"
@@ -1121,6 +1158,7 @@ export interface Store {
  * Beginning of processing a Sample.
  */
 export interface SampleInitEvent {
+  span_id: SpanId;
   timestamp: Timestamp;
   working_start: WorkingStart;
   pending: Pending;
@@ -1145,6 +1183,7 @@ export interface Sample {
  * The sample was unable to finish processing due to a limit
  */
 export interface SampleLimitEvent {
+  span_id: SpanId1;
   timestamp: Timestamp1;
   working_start: WorkingStart1;
   pending: Pending1;
@@ -1157,6 +1196,7 @@ export interface SampleLimitEvent {
  * Sandbox execution or I/O
  */
 export interface SandboxEvent {
+  span_id: SpanId2;
   timestamp: Timestamp2;
   working_start: WorkingStart2;
   pending: Pending2;
@@ -1174,6 +1214,7 @@ export interface SandboxEvent {
  * Change to the current `TaskState`
  */
 export interface StateEvent {
+  span_id: SpanId3;
   timestamp: Timestamp3;
   working_start: WorkingStart3;
   pending: Pending3;
@@ -1198,6 +1239,7 @@ export interface JsonChange {
  * Change to data within the current `Store`.
  */
 export interface StoreEvent {
+  span_id: SpanId4;
   timestamp: Timestamp4;
   working_start: WorkingStart4;
   pending: Pending4;
@@ -1208,6 +1250,7 @@ export interface StoreEvent {
  * Call to a language model.
  */
 export interface ModelEvent {
+  span_id: SpanId5;
   timestamp: Timestamp5;
   working_start: WorkingStart5;
   pending: Pending5;
@@ -1219,6 +1262,7 @@ export interface ModelEvent {
   tool_choice: ToolChoice;
   config: GenerateConfig;
   output: ModelOutput;
+  retries: Retries;
   error: Error1;
   cache: Cache;
   call: ModelCall | null;
@@ -1289,6 +1333,7 @@ export interface Response {
  * Call to a tool.
  */
 export interface ToolEvent {
+  span_id: SpanId6;
   timestamp: Timestamp6;
   working_start: WorkingStart6;
   pending: Pending6;
@@ -1315,6 +1360,7 @@ export interface Arguments1 {
  * Tool approval.
  */
 export interface ApprovalEvent {
+  span_id: SpanId7;
   timestamp: Timestamp7;
   working_start: WorkingStart7;
   pending: Pending7;
@@ -1341,6 +1387,7 @@ export interface ToolCallView {
  * Input screen interaction.
  */
 export interface InputEvent {
+  span_id: SpanId8;
   timestamp: Timestamp8;
   working_start: WorkingStart8;
   pending: Pending8;
@@ -1355,6 +1402,7 @@ export interface InputEvent {
  * resulting from a call to `score`.
  */
 export interface ScoreEvent {
+  span_id: SpanId9;
   timestamp: Timestamp9;
   working_start: WorkingStart9;
   pending: Pending9;
@@ -1367,6 +1415,7 @@ export interface ScoreEvent {
  * Event with sample error.
  */
 export interface ErrorEvent {
+  span_id: SpanId10;
   timestamp: Timestamp10;
   working_start: WorkingStart10;
   pending: Pending10;
@@ -1377,6 +1426,7 @@ export interface ErrorEvent {
  * Log message recorded with Python logger.
  */
 export interface LoggerEvent {
+  span_id: SpanId11;
   timestamp: Timestamp11;
   working_start: WorkingStart11;
   pending: Pending11;
@@ -1399,6 +1449,7 @@ export interface LoggingMessage {
  * Event with custom info/data.
  */
 export interface InfoEvent {
+  span_id: SpanId12;
   timestamp: Timestamp12;
   working_start: WorkingStart12;
   pending: Pending12;
@@ -1407,27 +1458,54 @@ export interface InfoEvent {
   data: JsonValue;
 }
 /**
- * Step within current sample or subtask.
+ * Mark the beginning of a transcript span.
  */
-export interface StepEvent {
+export interface SpanBeginEvent {
+  span_id: SpanId13;
   timestamp: Timestamp13;
   working_start: WorkingStart13;
   pending: Pending13;
   event: Event13;
-  action: Action1;
+  id: Id8;
+  parent_id: ParentId;
   type: Type13;
   name: Name11;
 }
 /**
- * Subtask spawned.
+ * Mark the end of a transcript span.
  */
-export interface SubtaskEvent {
+export interface SpanEndEvent {
+  span_id: SpanId14;
   timestamp: Timestamp14;
   working_start: WorkingStart14;
   pending: Pending14;
   event: Event14;
-  name: Name12;
+  id: Id9;
+}
+/**
+ * Step within current sample or subtask.
+ */
+export interface StepEvent {
+  span_id: SpanId15;
+  timestamp: Timestamp15;
+  working_start: WorkingStart15;
+  pending: Pending15;
+  event: Event15;
+  action: Action1;
   type: Type14;
+  name: Name12;
+}
+/**
+ * Subtask spawned.
+ */
+export interface SubtaskEvent {
+  span_id: SpanId16;
+  timestamp: Timestamp16;
+  working_start: WorkingStart16;
+  pending: Pending16;
+  event: Event16;
+  name: Name13;
+  type: Type15;
   input: Input5;
   result: Result2;
   events: Events2;
@@ -1450,7 +1528,7 @@ export interface Attachments {
  * Limit encontered by sample.
  */
 export interface EvalSampleLimit {
-  type: Type15;
+  type: Type16;
   limit: Limit2;
 }
 /**

inspect_ai/_view/www/src/app/log-view/navbar/SecondaryBar.tsx

@@ -8,7 +8,7 @@ import {
   EvalStats,
 } from "../../../@types/log";
 import { EvalDescriptor } from "../../../app/samples/descriptor/types";
-import { scoreFilterItems } from "../../../app/samples/sample-tools/filters";
+import { sampleFilterItems } from "../../../app/samples/sample-tools/filters";
 import { ExpandablePanel } from "../../../components/ExpandablePanel";
 import { LabeledValue } from "../../../components/LabeledValue";
 import { useEvalDescriptor } from "../../../state/hooks";
@@ -181,7 +181,7 @@ const ScorerSummary: FC<ScoreSummaryProps> = ({ evalDescriptor }) => {
     return null;
   }
 
-  const items = scoreFilterItems(evalDescriptor);
+  const items = sampleFilterItems(evalDescriptor);
   return (
     <span style={{ position: "relative" }}>
       {Array.from(items).map((item, index, array) => (

inspect_ai/_view/www/src/app/log-view/tabs/SamplesTab.tsx

@@ -50,10 +50,7 @@ export const useSamplesTabConfig = (
           : totalSampleCount === 1
             ? [<ScoreFilterTools />]
             : [
-                <SampleTools
-                  samples={sampleSummaries || []}
-                  key="sample-tools"
-                />,
+                <SampleTools key="sample-tools" />,
                 evalStatus === "started" && !streamSamples && (
                   <ToolButton
                     key="refresh"

inspect_ai/_view/www/src/app/samples/SamplesTools.tsx

@@ -1,6 +1,5 @@
 import { FC } from "react";
 import { Fragment } from "react/jsx-runtime";
-import { SampleSummary } from "../../client/api/types";
 import { useScore, useScores } from "../../state/hooks";
 import { useStore } from "../../state/store";
 import { EpochFilter } from "./sample-tools/EpochFilter";
@@ -8,16 +7,11 @@ import { SampleFilter } from "./sample-tools/sample-filter/SampleFilter";
 import { SelectScorer } from "./sample-tools/SelectScorer";
 import { SortFilter } from "./sample-tools/SortFilter";
 
-interface SampleToolsProps {
-  samples: SampleSummary[];
-}
+interface SampleToolsProps {}
 
-export const SampleTools: FC<SampleToolsProps> = ({ samples }) => {
+export const SampleTools: FC<SampleToolsProps> = () => {
   const selectedLogSummary = useStore((state) => state.log.selectedLogSummary);
 
-  const filter = useStore((state) => state.log.filter);
-  const setFilter = useStore((state) => state.logActions.setFilter);
-
   const scores = useScores();
   const score = useScore();
   const setScore = useStore((state) => state.logActions.setScore);
@@ -29,11 +23,7 @@ export const SampleTools: FC<SampleToolsProps> = ({ samples }) => {
   const epochs = selectedLogSummary?.eval.config.epochs || 1;
   return (
     <Fragment>
-      <SampleFilter
-        samples={samples}
-        scoreFilter={filter}
-        setScoreFilter={setFilter}
-      />
+      <SampleFilter />
       {scores?.length > 1 ? (
         <SelectScorer scores={scores} score={score} setScore={setScore} />
       ) : undefined}

inspect_ai/_view/www/src/app/samples/sample-tools/SelectScorer.tsx

@@ -1,7 +1,7 @@
 import clsx from "clsx";
 import { ScoreLabel } from "../../../app/types";
 
-import { ChangeEvent, FC, useCallback } from "react";
+import { ChangeEvent, FC, useCallback, useMemo } from "react";
 import styles from "./SelectScorer.module.css";
 
 interface SelectScorerProps {
@@ -15,23 +15,18 @@ export const SelectScorer: FC<SelectScorerProps> = ({
   score,
   setScore,
 }) => {
-  const scorers = scores.reduce((accum, scorer) => {
-    if (
-      !accum.find((sc) => {
-        return scorer.scorer === sc.scorer;
-      })
-    ) {
-      accum.push(scorer);
-    }
-    return accum;
-  }, [] as ScoreLabel[]);
-
-  const handleSelectScore = useCallback(
-    (index: number) => {
-      setScore(scores[index]);
-    },
-    [setScore, scores],
-  );
+  const scorers = useMemo(() => {
+    return scores.reduce((accum, scorer) => {
+      if (
+        !accum.find((sc) => {
+          return scorer.scorer === sc.scorer;
+        })
+      ) {
+        accum.push(scorer);
+      }
+      return accum;
+    }, [] as ScoreLabel[]);
+  }, [scores]);
 
   if (scorers.length === 1) {
     // There is only a single scorer in play, just show the list of available scores
@@ -50,8 +45,8 @@ export const SelectScorer: FC<SelectScorerProps> = ({
         </span>
         <ScoreSelector
           scores={scores}
-          selectedIndex={scoreIndex(scores, score)}
-          setSelectedIndex={handleSelectScore}
+          selectedScore={score}
+          setSelectedScore={setScore}
         />
       </div>
     );
@@ -79,15 +74,15 @@ export const SelectScorer: FC<SelectScorerProps> = ({
         </span>
         <ScorerSelector
           scorers={scorers}
-          selectedIndex={scorerIndex(scorers, score)}
-          setSelectedIndex={handleSelectScore}
+          selectedScore={score}
+          setSelectedScore={setScore}
         />
         {scorerScores.length > 1 ? (
           <ScoreSelector
             className={clsx(styles.secondSel)}
             scores={scorerScores}
-            selectedIndex={scoreIndex(scorerScores, score)}
-            setSelectedIndex={handleSelectScore}
+            selectedScore={score}
+            setSelectedScore={setScore}
           />
         ) : undefined}
       </div>
@@ -97,25 +92,33 @@ export const SelectScorer: FC<SelectScorerProps> = ({
 
 interface ScoreSelectorProps {
   scores: ScoreLabel[];
-  selectedIndex: number;
-  setSelectedIndex: (index: number) => void;
+  selectedScore?: ScoreLabel;
+  setSelectedScore: (score: ScoreLabel) => void;
   className?: string | string[];
 }
 
 const ScoreSelector: FC<ScoreSelectorProps> = ({
   scores,
-  selectedIndex,
-  setSelectedIndex,
+  selectedScore,
+  setSelectedScore,
   className,
 }) => {
   const handleChange = useCallback(
     (e: ChangeEvent<HTMLSelectElement>) => {
       const sel = e.target as HTMLSelectElement;
-      setSelectedIndex(sel.selectedIndex);
+      setSelectedScore(scores[sel.selectedIndex]);
     },
-    [setSelectedIndex],
+    [setSelectedScore, scores],
   );
 
+  const index = scores.findIndex((sc) => {
+    return (
+      selectedScore &&
+      sc.name === selectedScore.name &&
+      sc.scorer === selectedScore.scorer
+    );
+  });
+
   return (
     <select
       className={clsx(
@@ -125,7 +128,7 @@ const ScoreSelector: FC<ScoreSelectorProps> = ({
         className,
       )}
       aria-label=".select-scorer-label"
-      value={scores[selectedIndex].name}
+      value={scores[index].name}
       onChange={handleChange}
     >
       {scores.map((score) => {
@@ -141,28 +144,32 @@ const ScoreSelector: FC<ScoreSelectorProps> = ({
 
 interface ScorerSelectorProps {
   scorers: ScoreLabel[];
-  selectedIndex: number;
-  setSelectedIndex: (index: number) => void;
+  selectedScore?: ScoreLabel;
+  setSelectedScore: (score: ScoreLabel) => void;
 }
 
 const ScorerSelector: FC<ScorerSelectorProps> = ({
   scorers,
-  selectedIndex,
-  setSelectedIndex,
+  selectedScore,
+  setSelectedScore,
 }) => {
   const handleChange = useCallback(
     (e: ChangeEvent<HTMLSelectElement>) => {
       const sel = e.target as HTMLSelectElement;
-      setSelectedIndex(sel.selectedIndex);
+      setSelectedScore(scorers[sel.selectedIndex]);
     },
-    [setSelectedIndex],
+    [setSelectedScore, scorers],
   );
 
+  const index = scorers.findIndex((sc) => {
+    return selectedScore && sc.scorer === selectedScore.scorer;
+  });
+
   return (
     <select
       className={clsx("form-select", "form-select-sm", "text-size-smaller")}
       aria-label=".epoch-filter-label"
-      value={scorers[selectedIndex].scorer}
+      value={scorers[index].scorer}
       onChange={handleChange}
     >
       {scorers.map((scorer) => {
@@ -175,13 +182,3 @@ const ScorerSelector: FC<ScorerSelectorProps> = ({
     </select>
   );
 };
-
-const scoreIndex = (scores: ScoreLabel[], score?: ScoreLabel) =>
-  scores.findIndex((sc) => {
-    return score && sc.name === score.name && sc.scorer === score.scorer;
-  });
-
-const scorerIndex = (scores: ScoreLabel[], score?: ScoreLabel) =>
-  scores.findIndex((sc) => {
-    return score && sc.scorer === score.scorer;
-  });

inspect_ai/_view/www/src/app/samples/sample-tools/filters.ts

@@ -1,19 +1,12 @@
 import { compileExpression } from "filtrex";
 import { Scores1 } from "../../../@types/log";
-import { ScoreLabel } from "../../../app/types";
+import { FilterError, ScoreLabel } from "../../../app/types";
 import { SampleSummary } from "../../../client/api/types";
 import { kScoreTypeBoolean } from "../../../constants";
 import { inputString } from "../../../utils/format";
 import { EvalDescriptor, ScoreDescriptor } from "../descriptor/types";
 
-export interface FilterError {
-  from: number;
-  to: number;
-  message: string;
-  severity: "warning" | "error";
-}
-
-export interface ScoreFilterItem {
+export interface SampleFilterItem {
   shortName?: string;
   qualifiedName?: string;
   canonicalName: string;
@@ -120,10 +113,10 @@ const sampleVariables = (sample: SampleSummary): Record<string, unknown> => {
  * Child metrics are accessed using dot notation (e.g. `scorer_name.score_name`) or
  * directly by name when it is unique.
  */
-export const scoreFilterItems = (
+export const sampleFilterItems = (
   evalDescriptor: EvalDescriptor,
-): ScoreFilterItem[] => {
-  const items: ScoreFilterItem[] = [];
+): SampleFilterItem[] => {
+  const items: SampleFilterItem[] = [];
   const bannedShortNames = bannedShortScoreNames(evalDescriptor.scores);
   const valueToString = (value: unknown) =>
     typeof value === "string" ? `"${value}"` : String(value);
@@ -296,8 +289,13 @@ export const filterSamples = (
   evalDescriptor: EvalDescriptor,
   samples: SampleSummary[],
   filterValue: string,
-): { result: SampleSummary[]; error: FilterError | undefined } => {
-  var error = undefined;
+): {
+  result: SampleSummary[];
+  error: FilterError | undefined;
+  allErrors: boolean;
+} => {
+  let error = undefined;
+  let errorCount = 0;
   const result = samples.filter((sample) => {
     if (filterValue) {
       const { matches, error: sampleError } = filterExpression(
@@ -306,10 +304,13 @@ export const filterSamples = (
         filterValue,
       );
       error ||= sampleError;
+      if (sampleError) {
+        errorCount++;
+      }
       return matches;
     } else {
       return true;
     }
   });
-  return { result, error };
+  return { result, error, allErrors: errorCount === samples.length };
 };