inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/types/log.d.ts

@@ -112,6 +112,7 @@ export type Input =
       | ChatMessageAssistant
       | ChatMessageTool
     )[];
+export type Role = "system";
 export type Content =
   | string
   | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
@@ -127,18 +128,17 @@ export type Type4 = "video";
 export type Video = string;
 export type Format1 = "mp4" | "mpeg" | "mov";
 export type Source = ("input" | "generate") | null;
-export type Role = "system";
+export type Role1 = "user";
 export type Content1 =
   | string
   | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source1 = ("input" | "generate") | null;
-export type Role1 = "user";
 export type ToolCallId = string[] | null;
+export type Role2 = "assistant";
 export type Content2 =
   | string
   | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source2 = ("input" | "generate") | null;
-export type Role2 = "assistant";
 export type ToolCalls = ToolCall[] | null;
 export type Id1 = string;
 export type Function = string;
@@ -148,11 +148,11 @@ export type Title = string | null;
 export type Format2 = "text" | "markdown";
 export type Content3 = string;
 export type Reasoning = string | null;
+export type Role3 = "tool";
 export type Content4 =
   | string
   | (ContentText | ContentImage | ContentAudio | ContentVideo)[];
 export type Source3 = ("input" | "generate") | null;
-export type Role3 = "tool";
 export type ToolCallId1 = string | null;
 export type Function1 = string | null;
 export type Type6 =
@@ -315,6 +315,7 @@ export type Timestamp8 = string;
 export type Pending8 = boolean | null;
 export type Event8 = "score";
 export type Target2 = string | string[] | null;
+export type Intermediate = boolean;
 export type Timestamp9 = string;
 export type Pending9 = boolean | null;
 export type Event9 = "error";
@@ -324,6 +325,7 @@ export type Event10 = "logger";
 export type Name7 = string | null;
 export type Level =
   | "debug"
+  | "trace"
   | "http"
   | "sandbox"
   | "info"
@@ -338,6 +340,7 @@ export type Lineno = number;
 export type Timestamp11 = string;
 export type Pending11 = boolean | null;
 export type Event11 = "info";
+export type Source4 = string | null;
 export type Timestamp12 = string;
 export type Pending12 = boolean | null;
 export type Event12 = "step";
@@ -423,6 +426,9 @@ export type SampleId1 = string | number | null;
 export type Samples2 = EvalSampleScore[];
 export type Location1 = string;
 
+/**
+ * Evaluation log.
+ */
 export interface EvalLog {
   version?: Version;
   status?: Status;
@@ -435,6 +441,9 @@ export interface EvalLog {
   reductions?: Reductions;
   location?: Location1;
 }
+/**
+ * Eval target and configuration.
+ */
 export interface EvalSpec {
   run_id: RunId;
   created: Created;
@@ -459,6 +468,9 @@ export interface EvalSpec {
 }
 export interface TaskAttribs {}
 export interface TaskArgs {}
+/**
+ * Dataset used for evaluation.
+ */
 export interface EvalDataset {
   name: Name;
   location: Location;
@@ -467,6 +479,9 @@ export interface EvalDataset {
   shuffled: Shuffled;
 }
 export interface ModelArgs {}
+/**
+ * Configuration used for evaluation.
+ */
 export interface EvalConfig {
   limit: Limit;
   sample_id: SampleId;
@@ -512,6 +527,9 @@ export interface ApproverPolicyConfig {
   params: Params;
 }
 export interface Params {}
+/**
+ * Git revision for evaluation.
+ */
 export interface EvalRevision {
   type: Type;
   origin: Origin;
@@ -520,19 +538,25 @@ export interface EvalRevision {
 export interface Packages {
   [k: string]: string;
 }
+/**
+ * Plan (solvers) used in evaluation.
+ */
 export interface EvalPlan {
   name: Name2;
   steps: Steps;
   finish: EvalPlanStep | null;
   config: GenerateConfig;
 }
+/**
+ * Solver step.
+ */
 export interface EvalPlanStep {
   solver: Solver1;
   params: Params1;
 }
 export interface Params1 {}
 /**
- * Base class for model generation configs.
+ * Model generation options.
  */
 export interface GenerateConfig {
   max_retries: MaxRetries;
@@ -559,12 +583,18 @@ export interface GenerateConfig {
   reasoning_effort: ReasoningEffort;
   reasoning_history: ReasoningHistory;
 }
+/**
+ * Scoring results from evaluation.
+ */
 export interface EvalResults {
   total_samples: TotalSamples;
   completed_samples: CompletedSamples;
   scores: Scores;
   metadata: Metadata3;
 }
+/**
+ * Score for evaluation task.
+ */
 export interface EvalScore {
   name: Name3;
   scorer: Scorer;
@@ -577,13 +607,19 @@ export interface Params2 {}
 export interface Metrics {
   [k: string]: EvalMetric;
 }
+/**
+ * Metric for evaluation score.
+ */
 export interface EvalMetric {
   name: Name4;
   value: Value;
-  options: Options;
+  params: Params3;
   metadata: Metadata1;
 }
-export interface Options {}
+export interface Params3 {}
+/**
+ * Timing and usage statistics.
+ */
 export interface EvalStats {
   started_at: StartedAt;
   completed_at: CompletedAt;
@@ -592,6 +628,9 @@ export interface EvalStats {
 export interface ModelUsage {
   [k: string]: ModelUsage1;
 }
+/**
+ * Token usage for completion.
+ */
 export interface ModelUsage1 {
   input_tokens: InputTokens;
   output_tokens: OutputTokens;
@@ -599,11 +638,17 @@ export interface ModelUsage1 {
   input_tokens_cache_write: InputTokensCacheWrite;
   input_tokens_cache_read: InputTokensCacheRead;
 }
+/**
+ * Eval error details.
+ */
 export interface EvalError {
   message: Message;
   traceback: Traceback;
   traceback_ansi: TracebackAnsi;
 }
+/**
+ * Sample from evaluation task.
+ */
 export interface EvalSample {
   id: Id;
   epoch: Epoch;
@@ -624,40 +669,61 @@ export interface EvalSample {
   attachments: Attachments;
   limit: EvalSampleLimit | null;
 }
+/**
+ * System chat message.
+ */
 export interface ChatMessageSystem {
+  role: Role;
   content: Content;
   source: Source;
-  role: Role;
 }
+/**
+ * Text content.
+ */
 export interface ContentText {
   type: Type1;
   text: Text;
 }
+/**
+ * Image content.
+ */
 export interface ContentImage {
   type: Type2;
   image: Image;
   detail: Detail;
 }
+/**
+ * Audio content.
+ */
 export interface ContentAudio {
   type: Type3;
   audio: Audio;
   format: Format;
 }
+/**
+ * Video content.
+ */
 export interface ContentVideo {
   type: Type4;
   video: Video;
   format: Format1;
 }
+/**
+ * User chat message.
+ */
 export interface ChatMessageUser {
+  role: Role1;
   content: Content1;
   source: Source1;
-  role: Role1;
   tool_call_id: ToolCallId;
 }
+/**
+ * Assistant chat message.
+ */
 export interface ChatMessageAssistant {
+  role: Role2;
   content: Content2;
   source: Source2;
-  role: Role2;
   tool_calls: ToolCalls;
   reasoning: Reasoning;
 }
@@ -678,10 +744,13 @@ export interface ToolCallContent {
   format: Format2;
   content: Content3;
 }
+/**
+ * Tool chat message.
+ */
 export interface ChatMessageTool {
+  role: Role3;
   content: Content4;
   source: Source3;
-  role: Role3;
   tool_call_id: ToolCallId1;
   function: Function1;
   error: ToolCallError | null;
@@ -690,6 +759,9 @@ export interface ToolCallError {
   type: Type6;
   message: Message1;
 }
+/**
+ * Output from model generation.
+ */
 export interface ModelOutput {
   model: Model1;
   choices: Choices1;
@@ -698,6 +770,9 @@ export interface ModelOutput {
   metadata: Metadata4;
   error: Error;
 }
+/**
+ * Choice generated for completion.
+ */
 export interface ChatCompletionChoice {
   message: ChatMessageAssistant;
   stop_reason: StopReason;
@@ -728,12 +803,6 @@ export interface TopLogprob {
 }
 /**
  * Score generated by a scorer.
- *
- * Args:
- *    value (Value): Score value.
- *    answer (str | None): Answer extracted from model output (optional).
- *    explanation (str | None): Explanation of score (optional).
- *    metadata (dict[str,Any]): Additional metadata related to the score.
  */
 export interface Score {
   value: Value1;
@@ -753,6 +822,9 @@ export interface SampleInitEvent {
   sample: Sample;
   state: JsonValue;
 }
+/**
+ * Sample for an evaluation task.
+ */
 export interface Sample {
   input: Input1;
   choices: Choices2;
@@ -887,7 +959,7 @@ export interface ToolFunction {
   name: Name6;
 }
 /**
- * Base class for model generation configs.
+ * Model generation options.
  */
 export interface GenerateConfig1 {
   max_retries: MaxRetries;
@@ -983,7 +1055,10 @@ export interface InputEvent {
   input_ansi: InputAnsi;
 }
 /**
- * Event with sample score.
+ * Event with score.
+ *
+ * Can be the final score for a `Sample`, or can be an intermediate score
+ * resulting from a call to `score`.
  */
 export interface ScoreEvent {
   timestamp: Timestamp8;
@@ -991,6 +1066,7 @@ export interface ScoreEvent {
   event: Event8;
   score: Score;
   target: Target2;
+  intermediate: Intermediate;
 }
 /**
  * Event with sample error.
@@ -1010,6 +1086,9 @@ export interface LoggerEvent {
   event: Event10;
   message: LoggingMessage;
 }
+/**
+ * Message written to Python log.
+ */
 export interface LoggingMessage {
   name: Name7;
   level: Level;
@@ -1026,6 +1105,7 @@ export interface InfoEvent {
   timestamp: Timestamp11;
   pending: Pending11;
   event: Event11;
+  source: Source4;
   data: JsonValue;
 }
 /**
@@ -1062,15 +1142,24 @@ export interface ModelUsage2 {
 export interface Attachments {
   [k: string]: string;
 }
+/**
+ * Limit encontered by sample.
+ */
 export interface EvalSampleLimit {
   type: Type13;
   limit: Limit2;
 }
+/**
+ * Score reductions.
+ */
 export interface EvalSampleReductions {
   scorer: Scorer1;
   reducer: Reducer1;
   samples: Samples2;
 }
+/**
+ * Score and sample_id scored.
+ */
 export interface EvalSampleScore {
   value: Value2;
   answer: Answer1;

inspect_ai/_view/www/src/types/prism.d.ts

@@ -1,4 +1,15 @@
 declare var Prism: {
   languages: any;
   highlight(contents: any, tokens: any, type: any): string;
+  highlightElement(
+    element: HTMLElement,
+    async?: boolean,
+    callback?: (element: HTMLElement) => void,
+  );
+
+  highlightAllUnder(
+    element: HTMLElement,
+    async?: boolean,
+    callback?: (element: HTMLElement) => void,
+  );
 };

inspect_ai/_view/www/src/types.ts

@@ -0,0 +1,71 @@
+import {
+  EvalLogHeader,
+  EvalSummary,
+  LogFiles,
+  SampleSummary,
+} from "./api/types";
+import { ContentImage, ContentText, EvalSample } from "./types/log";
+
+export interface ApplicationState {
+  logs?: LogFiles;
+  selectedLogIndex?: number;
+  logHeaders?: Record<string, EvalLogHeader>;
+  headersLoading?: boolean;
+  selectedLog?: CurrentLog;
+  selectedWorkspaceTab?: string;
+  selectedSampleIndex?: number;
+  selectedSample?: EvalSample;
+  sampleStatus?: "loading" | "ok" | "error";
+  sampleError?: Error;
+  selectedSampleTab?: string;
+  sampleScrollPosition?: number;
+  showingSampleDialog?: boolean;
+  status?: AppStatus;
+  offcanvas?: boolean;
+  showFind?: boolean;
+  filter?: ScoreFilter;
+  epoch?: string;
+  sort?: string;
+  scores?: ScoreLabel[];
+  score?: ScoreLabel;
+  filteredSamples?: SampleSummary[];
+  groupBy?: "none" | "epoch" | "sample";
+  groupByOrder?: "asc" | "desc";
+  workspaceTabScrollPosition?: Record<string, number>;
+}
+
+export interface AppStatus {
+  loading: boolean;
+  error?: Error;
+}
+
+export interface Capabilities {
+  downloadFiles: boolean;
+  webWorkers: boolean;
+}
+
+export interface CurrentLog {
+  name: string;
+  contents: EvalSummary;
+}
+
+export interface Logs {
+  log_dir: string;
+  files: string[];
+}
+
+export interface ScoreLabel {
+  name: string;
+  scorer: string;
+}
+
+export interface ScoreFilter {
+  value?: string;
+}
+
+export type SampleMode = "none" | "single" | "many";
+
+export interface ContentTool {
+  type: "tool";
+  content: (ContentImage | ContentText)[];
+}

inspect_ai/_view/www/src/usage/ModelTokenTable.tsx

@@ -0,0 +1,28 @@
+import { TokenHeader, TokenRow, TokenTable } from "./TokenTable";
+
+interface ModelTokenTable {
+  model_usage: any;
+  className?: string | string[];
+}
+
+export const ModelTokenTable: React.FC<ModelTokenTable> = ({
+  model_usage,
+  className,
+}) => {
+  return (
+    <TokenTable className={className}>
+      <TokenHeader />
+      <tbody>
+        {Object.keys(model_usage).map((key) => {
+          return (
+            <TokenRow
+              key={key}
+              model={`${key}-token-row`}
+              usage={model_usage[key]}
+            />
+          );
+        })}
+      </tbody>
+    </TokenTable>
+  );
+};

inspect_ai/_view/www/src/usage/ModelUsagePanel.module.css

@@ -0,0 +1,24 @@
+.wrapper {
+  display: grid;
+  grid-template-columns: 0 auto auto;
+  column-gap: 1.5em;
+  row-gap: 0.2em;
+}
+
+.col2 {
+  grid-column: 2;
+}
+
+.col1_3 {
+  grid-column: 1/3;
+}
+
+.col3 {
+  grid-column: 3;
+}
+
+.separator {
+  grid-column: -1/1;
+  height: 1px;
+  background-color: var(--bs-light-border-subtle);
+}

inspect_ai/_view/www/src/usage/ModelUsagePanel.tsx

@@ -0,0 +1,97 @@
+import clsx from "clsx";
+import { Fragment } from "react";
+import { ModelUsage1 } from "../types/log";
+import { formatNumber } from "../utils/format";
+import styles from "./ModelUsagePanel.module.css";
+
+interface ModelUsageProps {
+  usage: ModelUsage1;
+}
+
+interface ModelUsageRow {
+  label: string | "---";
+  value?: number;
+  secondary?: boolean;
+  bordered?: boolean;
+}
+
+/**
+ * Renders the ModelUsagePanel component.
+ */
+export const ModelUsagePanel: React.FC<ModelUsageProps> = ({ usage }) => {
+  if (!usage) {
+    return null;
+  }
+
+  const rows: ModelUsageRow[] = [
+    {
+      label: "input",
+      value: usage.input_tokens,
+      secondary: false,
+    },
+  ];
+
+  if (usage.input_tokens_cache_read) {
+    rows.push({
+      label: "cache_read",
+      value: usage.input_tokens_cache_read,
+      secondary: true,
+    });
+  }
+
+  if (usage.input_tokens_cache_write) {
+    rows.push({
+      label: "cache_write",
+      value: usage.input_tokens_cache_write,
+      secondary: true,
+    });
+  }
+
+  rows.push({
+    label: "Output",
+    value: usage.output_tokens,
+    secondary: false,
+    bordered: true,
+  });
+
+  rows.push({
+    label: "---",
+    value: undefined,
+    secondary: false,
+  });
+
+  rows.push({
+    label: "Total",
+    value: usage.total_tokens,
+    secondary: false,
+  });
+
+  return (
+    <div className={clsx("text-size-small", styles.wrapper)}>
+      {rows.map((row, idx) => {
+        if (row.label === "---") {
+          return (
+            <div key={`$usage-sep-${idx}`} className={styles.separator}></div>
+          );
+        } else {
+          return (
+            <Fragment key={`$usage-row-${idx}`}>
+              <div
+                className={clsx(
+                  "text-style-label",
+                  "text-style-secondary",
+                  row.secondary ? styles.col2 : styles.col1_3,
+                )}
+              >
+                {row.label}
+              </div>
+              <div className={styles.col3}>
+                {row.value ? formatNumber(row.value) : ""}
+              </div>
+            </Fragment>
+          );
+        }
+      })}
+    </div>
+  );
+};

inspect_ai/_view/www/src/usage/TokenTable.module.css

@@ -0,0 +1,17 @@
+.table {
+  width: 100%;
+  margin-top: 0.7rem;
+}
+
+.tableTokens {
+  padding-bottom: 0.7rem;
+}
+
+.tableH {
+  padding: 0;
+  font-weight: 300;
+}
+
+.model {
+  padding-right: 1em;
+}