inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/api/api-browser.ts

@@ -1,7 +1,12 @@
-import { Capabilities } from "../types";
 import { asyncJsonParse } from "../utils/json-worker";
 import { download_file } from "./api-shared";
-import { LogContents, LogViewAPI } from "./types";
+import {
+  Capabilities,
+  LogContents,
+  LogViewAPI,
+  PendingSampleResponse,
+  SampleDataResponse,
+} from "./types";
 
 const loaded_time = Date.now();
 let last_eval_time = 0;
@@ -49,24 +54,188 @@ async function eval_log_headers(files: string[]) {
   return (await api("GET", `/api/log-headers?${params.toString()}`)).parsed;
 }
 
+async function eval_pending_samples(
+  log_file: string,
+  etag?: string,
+): Promise<PendingSampleResponse> {
+  // Attach the log file
+  const params = new URLSearchParams();
+  params.append("log", log_file);
+
+  // Send the etag along
+  const headers: Record<string, string> = {};
+  if (etag) {
+    headers["If-None-Match"] = etag;
+  }
+
+  // Build up the request
+  const request: Request<PendingSampleResponse> = {
+    headers,
+    parse: async (text: string) => {
+      const pendingSamples = await asyncJsonParse(text);
+      return {
+        status: "OK",
+        pendingSamples,
+      };
+    },
+    handleError: (status: number) => {
+      if (status === 404) {
+        return {
+          status: "NotFound",
+        };
+      } else if (status === 304) {
+        return {
+          status: "NotModified",
+        };
+      }
+    },
+  };
+  // Fetch the result
+  const result = (
+    await apiRequest<PendingSampleResponse>(
+      "GET",
+      `/api/pending-samples?${params.toString()}`,
+      request,
+    )
+  ).parsed;
+
+  return result;
+}
+
+async function eval_log_sample_data(
+  log_file: string,
+  id: string | number,
+  epoch: number,
+  last_event?: number,
+  last_attachment?: number,
+): Promise<SampleDataResponse | undefined> {
+  const params = new URLSearchParams();
+  params.append("log", log_file);
+  params.append("id", String(id));
+  params.append("epoch", String(epoch));
+  if (last_event) {
+    params.append("last-event-id", String(last_event));
+  }
+
+  if (last_attachment) {
+    params.append("after-attachment-id", String(last_attachment));
+  }
+
+  // Build up the request
+  const request: Request<SampleDataResponse> = {
+    headers: {},
+    parse: async (text: string) => {
+      const pendingSamples = await asyncJsonParse(text);
+      return {
+        status: "OK",
+        sampleData: pendingSamples,
+      };
+    },
+    handleError: (status: number) => {
+      if (status === 404) {
+        return {
+          status: "NotFound",
+        };
+      } else if (status === 304) {
+        return {
+          status: "NotModified",
+        };
+      }
+    },
+  };
+  // Fetch the result
+  const result = (
+    await apiRequest<SampleDataResponse>(
+      "GET",
+      `/api/pending-sample-data?${params.toString()}`,
+      request,
+    )
+  ).parsed;
+
+  return result;
+}
+
+interface Request<T> {
+  headers?: Record<string, string>;
+  body?: string;
+  parse?: (text: string) => Promise<T>;
+  handleError?: (status: number) => T | undefined;
+}
+
+async function apiRequest<T>(
+  method: "GET" | "POST" | "PUT" | "DELETE",
+  path: string,
+  request: Request<T>,
+): Promise<{ raw: string; parsed: T }> {
+  // build headers
+  const responseHeaders: HeadersInit = {
+    Accept: "application/json",
+    Pragma: "no-cache",
+    Expires: "0",
+    ["Cache-Control"]: "no-cache",
+    ...request.headers,
+  };
+  if (request.body) {
+    responseHeaders["Content-Type"] = "application/json";
+  }
+
+  // make request
+  const response = await fetch(`${path}`, {
+    method,
+    headers: responseHeaders,
+    body: request.body,
+  });
+  if (response.ok) {
+    const text = await response.text();
+    const parse = request.parse || asyncJsonParse;
+    return {
+      parsed: (await parse(text)) as T,
+      raw: text,
+    };
+  } else if (response.status !== 200) {
+    // See if the request handler wants to handle this
+    const errorResponse = request.handleError
+      ? request.handleError(response.status)
+      : undefined;
+    if (errorResponse) {
+      return {
+        raw: response.statusText,
+        parsed: errorResponse,
+      };
+    }
+
+    const message = (await response.text()) || response.statusText;
+    const error = new Error(`Error: ${response.status}: ${message})`);
+    throw error;
+  } else {
+    throw new Error(`${response.status} - ${response.statusText} `);
+  }
+}
+
 async function api(
   method: "GET" | "POST" | "PUT" | "DELETE",
   path: string,
+  headers?: Record<string, string>,
   body?: string,
 ) {
   // build headers
-  const headers: HeadersInit = {
+  const responseHeaders: HeadersInit = {
     Accept: "application/json",
     Pragma: "no-cache",
     Expires: "0",
     ["Cache-Control"]: "no-cache",
+    ...headers,
   };
   if (body) {
-    headers["Content-Type"] = "application/json";
+    responseHeaders["Content-Type"] = "application/json";
   }
 
   // make request
-  const response = await fetch(`${path}`, { method, headers, body });
+  const response = await fetch(`${path}`, {
+    method,
+    headers: responseHeaders,
+    body,
+  });
   if (response.ok) {
     const text = await response.text();
     return {
@@ -121,5 +290,7 @@ const browserApi: LogViewAPI = {
   eval_log_headers,
   download_file,
   open_log_file,
+  eval_pending_samples,
+  eval_log_sample_data,
 };
 export default browserApi;

inspect_ai/_view/www/src/api/api-vscode.ts

@@ -8,9 +8,20 @@ import {
   kMethodEvalLogHeaders,
   kMethodEvalLogs,
   kMethodEvalLogSize,
+  kMethodPendingSamples,
+  kMethodSampleData,
   webViewJsonRpcClient,
 } from "./jsonrpc";
-import { Capabilities, LogContents, LogViewAPI } from "./types";
+import {
+  Capabilities,
+  LogContents,
+  LogViewAPI,
+  PendingSampleResponse,
+  SampleDataResponse,
+} from "./types";
+
+const kNotFoundSignal = "NotFound";
+const kNotModifiedSignal = "NotModified";
 
 const vscodeClient = webViewJsonRpcClient(getVscodeApi());
 
@@ -75,6 +86,67 @@ async function eval_log_headers(files: string[]) {
   }
 }
 
+async function eval_pending_samples(
+  log_file: string,
+  etag?: string,
+): Promise<PendingSampleResponse> {
+  // TODO: use web worked to parse when possible
+  const response = await vscodeClient(kMethodPendingSamples, [log_file, etag]);
+  if (response) {
+    if (response === kNotModifiedSignal) {
+      return {
+        status: "NotModified",
+      };
+    } else if (response === kNotFoundSignal) {
+      return {
+        status: "NotFound",
+      };
+    }
+
+    const json = await asyncJsonParse(response);
+    return {
+      status: "OK",
+      pendingSamples: json,
+    };
+  } else {
+    throw new Error(`Unable to load pending samples ${log_file}.`);
+  }
+}
+
+async function eval_log_sample_data(
+  log_file: string,
+  id: string | number,
+  epoch: number,
+  last_event?: number,
+  last_attachment?: number,
+): Promise<SampleDataResponse | undefined> {
+  const response = await vscodeClient(kMethodSampleData, [
+    log_file,
+    id,
+    epoch,
+    last_event,
+    last_attachment,
+  ]);
+  if (response) {
+    if (response === kNotModifiedSignal) {
+      return {
+        status: "NotModified",
+      };
+    } else if (response === kNotFoundSignal) {
+      return {
+        status: "NotFound",
+      };
+    }
+    const json = await asyncJsonParse(response);
+    return {
+      status: "OK",
+      sampleData: json,
+    };
+  } else {
+    throw new Error(`Unable to load live sample data ${log_file}.`);
+  }
+}
+
 async function download_file() {
   throw Error("Downloading files is not supported in VS Code");
 }
@@ -97,6 +169,8 @@ const api: LogViewAPI = {
   eval_log_headers,
   download_file,
   open_log_file,
+  eval_pending_samples,
+  eval_log_sample_data,
 };
 
 export default api;

inspect_ai/_view/www/src/api/client-api.ts

@@ -1,4 +1,8 @@
-import { openRemoteLogFile, RemoteLogFile } from "../logfile/remoteLogFile";
+import {
+  openRemoteLogFile,
+  RemoteLogFile,
+  SampleNotFoundError,
+} from "../logfile/remoteLogFile";
 import { FileSizeLimitError } from "../logfile/remoteZipFile";
 import { EvalLog, EvalSample } from "../types/log";
 import { encodePathParts } from "./api-shared";
@@ -8,6 +12,8 @@ import {
   LogContents,
   LogFiles,
   LogViewAPI,
+  PendingSampleResponse,
+  SampleDataResponse,
 } from "./types";
 
 const isEvalFile = (file: string) => {
@@ -156,19 +162,34 @@ export const clientApi = (api: LogViewAPI, log_file?: string): ClientAPI => {
     epoch: number,
   ): Promise<EvalSample | undefined> => {
     if (isEvalFile(log_file)) {
-      const remoteLogFile = await remoteEvalFile(log_file, true);
-      try {
-        if (remoteLogFile) {
-          const sample = await remoteLogFile.readSample(String(id), epoch);
-          return sample;
-        } else {
-          throw new Error(`Unable to read remove eval file ${log_file}`);
+      async function fetchSample(useCache: boolean) {
+        const remoteLogFile = await remoteEvalFile(log_file, useCache);
+        if (!remoteLogFile) {
+          throw new Error(`Unable to read remote eval file ${log_file}`);
         }
-      } catch (error) {
+        return await remoteLogFile.readSample(String(id), epoch);
+      }
+
+      function handleError(error: unknown) {
         if (error instanceof FileSizeLimitError) {
           throw new SampleSizeLimitedExceededError(id, epoch, error.maxBytes);
+        }
+        throw error;
+      }
+
+      try {
+        // First attempt with cache
+        return await fetchSample(true);
+      } catch (error) {
+        if (error instanceof SampleNotFoundError) {
+          try {
+            // Retry without cache
+            return await fetchSample(false);
+          } catch (retryError) {
+            handleError(retryError);
+          }
         } else {
-          throw error;
+          handleError(error);
         }
       }
     } else {
@@ -263,6 +284,35 @@ export const clientApi = (api: LogViewAPI, log_file?: string): ClientAPI => {
     throw new Error("Unable to determine log paths.");
   };
 
+  const get_log_pending_samples = (
+    log_file: string,
+    etag?: string,
+  ): Promise<PendingSampleResponse> => {
+    if (!api.eval_pending_samples) {
+      throw new Error("API doesn't support streamed samples");
+    }
+    return api.eval_pending_samples(log_file, etag);
+  };
+
+  const get_log_sample_data = (
+    log_file: string,
+    id: string | number,
+    epoch: number,
+    last_event?: number,
+    last_attachment?: number,
+  ): Promise<SampleDataResponse | undefined> => {
+    if (!api.eval_log_sample_data) {
+      throw new Error("API doesn't supported streamed sample data");
+    }
+    return api.eval_log_sample_data(
+      log_file,
+      id,
+      epoch,
+      last_event,
+      last_attachment,
+    );
+  };
+
   return {
     client_events: () => {
       return api.client_events();
@@ -284,5 +334,11 @@ export const clientApi = (api: LogViewAPI, log_file?: string): ClientAPI => {
     ) => {
       return api.download_file(download_file, file_contents);
     },
+    get_log_pending_samples: api.eval_pending_samples
+      ? get_log_pending_samples
+      : undefined,
+    get_log_sample_data: api.eval_log_sample_data
+      ? get_log_sample_data
+      : undefined,
   };
 };

inspect_ai/_view/www/src/api/jsonrpc.ts

@@ -39,6 +39,8 @@ export const kMethodEvalLog = "eval_log";
 export const kMethodEvalLogSize = "eval_log_size";
 export const kMethodEvalLogBytes = "eval_log_bytes";
 export const kMethodEvalLogHeaders = "eval_log_headers";
+export const kMethodPendingSamples = "eval_log_pending_samples";
+export const kMethodSampleData = "eval_log_sample_data";
 
 export const kJsonRpcParseError = -32700;
 export const kJsonRpcInvalidRequest = -32600;

inspect_ai/_view/www/src/api/types.ts

@@ -1,4 +1,5 @@
 import {
+  ApprovalEvent,
   EvalError,
   EvalLog,
   EvalPlan,
@@ -6,11 +7,22 @@ import {
   EvalSample,
   EvalSpec,
   EvalStats,
+  InfoEvent,
   Input,
+  LoggerEvent,
+  ModelEvent,
+  SampleInitEvent,
+  SampleLimitEvent,
+  SandboxEvent,
+  ScoreEvent,
   Scores1,
+  StateEvent,
   Status,
+  StepEvent,
+  StoreEvent,
+  SubtaskEvent,
   Target,
-  Type11,
+  ToolEvent,
   Version,
 } from "../types/log";
 
@@ -25,6 +37,66 @@ export interface EvalSummary {
   sampleSummaries: SampleSummary[];
 }
 
+export interface PendingSampleResponse {
+  pendingSamples?: PendingSamples;
+  status: "NotModified" | "NotFound" | "OK";
+}
+
+export interface SampleDataResponse {
+  sampleData?: SampleData;
+  status: "NotModified" | "NotFound" | "OK";
+}
+
+export interface RunningMetric {
+  scorer: string;
+  name: string;
+  value?: number;
+  reducer?: string;
+}
+
+export interface PendingSamples {
+  metrics?: RunningMetric[];
+  samples: SampleSummary[];
+  refresh: number;
+  etag?: string;
+}
+
+export interface SampleData {
+  events: EventData[];
+  attachments: AttachmentData[];
+}
+
+export interface EventData {
+  id: number;
+  event_id: string;
+  sample_id: string;
+  epoch: number;
+  event:
+    | SampleInitEvent
+    | SampleLimitEvent
+    | SandboxEvent
+    | StateEvent
+    | StoreEvent
+    | ModelEvent
+    | ToolEvent
+    | ApprovalEvent
+    | InputEvent
+    | ScoreEvent
+    | ErrorEvent
+    | LoggerEvent
+    | InfoEvent
+    | StepEvent
+    | SubtaskEvent;
+}
+
+export interface AttachmentData {
+  id: number;
+  sample_id: string;
+  epoch: number;
+  hash: string;
+  content: string;
+}
+
 export interface EvalLogHeader {
   version?: Version;
   status?: Status;
@@ -42,7 +114,8 @@ export interface SampleSummary {
   target: Target;
   scores: Scores1;
   error?: string;
-  limit?: Type11;
+  limit?: string;
+  completed?: boolean;
 }
 
 export interface BasicSampleData {
@@ -55,6 +128,9 @@ export interface BasicSampleData {
 export interface Capabilities {
   downloadFiles: boolean;
   webWorkers: boolean;
+  streamSamples: boolean;
+  streamSampleData: boolean;
+  nativeFind: boolean;
 }
 
 export interface LogViewAPI {
@@ -77,6 +153,17 @@ export interface LogViewAPI {
     filecontents: string | Blob | ArrayBuffer | ArrayBufferView,
   ) => Promise<void>;
   open_log_file: (logFile: string, log_dir: string) => Promise<void>;
+  eval_pending_samples?: (
+    log_file: string,
+    etag?: string,
+  ) => Promise<PendingSampleResponse>;
+  eval_log_sample_data?: (
+    log_file: string,
+    id: string | number,
+    epoch: number,
+    last_event?: number,
+    last_attachment?: number,
+  ) => Promise<SampleDataResponse | undefined>;
 }
 
 export interface ClientAPI {
@@ -94,6 +181,24 @@ export interface ClientAPI {
     file_contents: string | Blob | ArrayBuffer | ArrayBufferView,
   ) => Promise<void>;
   open_log_file: (log_file: string, log_dir: string) => Promise<void>;
+
+  get_log_pending_samples?: (
+    log_file: string,
+    etag?: string,
+  ) => Promise<PendingSampleResponse>;
+  get_log_sample_data?: (
+    log_file: string,
+    id: string | number,
+    epoch: number,
+    last_event?: number,
+    last_attachment?: number,
+  ) => Promise<SampleDataResponse | undefined>;
+}
+
+export interface ClientStorage {
+  getItem: (name: string) => unknown;
+  setItem: (name: string, value: unknown) => void;
+  removeItem: (name: string) => void;
 }
 
 export interface FetchResponse {

inspect_ai/_view/www/src/appearance/icons.ts

@@ -66,11 +66,13 @@ export const ApplicationIcons = {
   menu: "bi bi-list",
   messages: "bi bi-chat-right-text",
   metadata: "bi bi-table",
+  metrics: "bi bi-clipboard-data",
   model: "bi bi-grid-3x3-gap",
   "toggle-right": "bi bi-chevron-right",
   more: "bi bi-zoom-in",
   "multiple-choice": "bi bi-card-list",
   next: "bi bi-chevron-right",
+  noSamples: "bi bi-ban",
   play: "bi bi-play-fill",
   previous: "bi bi-chevron-left",
   refresh: "bi bi-arrow-clockwise",

inspect_ai/_view/www/src/components/AsciinemaPlayer.tsx

@@ -1,6 +1,6 @@
 import * as AsciicinemaPlayerJS from "asciinema-player";
 import "asciinema-player/dist/bundle/asciinema-player.css";
-import React, { useEffect, useRef } from "react";
+import { CSSProperties, FC, useEffect, useRef } from "react";
 
 interface AsciinemaPlayerProps {
   id?: string;
@@ -10,7 +10,7 @@ interface AsciinemaPlayerProps {
   rows?: number;
   cols?: number;
   fit?: string;
-  style?: React.CSSProperties;
+  style?: CSSProperties;
   speed?: number;
   autoPlay?: boolean;
   loop?: boolean;
@@ -19,7 +19,7 @@ interface AsciinemaPlayerProps {
   className?: string;
 }
 
-export const AsciinemaPlayer: React.FC<AsciinemaPlayerProps> = ({
+export const AsciinemaPlayer: FC<AsciinemaPlayerProps> = ({
   id,
   rows,
   cols,

inspect_ai/_view/www/src/components/Card.tsx

@@ -14,11 +14,13 @@ interface CardHeaderProps {
 interface CardBodyProps {
   id?: string;
   children?: ReactNode;
+  className?: string | string[];
 }
 
 interface CardProps {
   id?: string;
   children?: ReactNode;
+  className?: string | string[];
 }
 
 interface CardCollapsingHeaderProps {
@@ -51,17 +53,17 @@ export const CardHeader: FC<CardHeaderProps> = ({
   );
 };
 
-export const CardBody: FC<CardBodyProps> = ({ id, children }) => {
+export const CardBody: FC<CardBodyProps> = ({ id, children, className }) => {
   return (
-    <div className={"card-body"} id={id || ""}>
+    <div className={clsx("card-body", className)} id={id || ""}>
       {children}
     </div>
   );
 };
 
-export const Card: FC<CardProps> = ({ id, children }) => {
+export const Card: FC<CardProps> = ({ id, children, className }) => {
   return (
-    <div className={"card"} id={id}>
+    <div className={clsx("card", className)} id={id}>
       {children}
     </div>
   );

inspect_ai/_view/www/src/components/DownloadPanel.tsx

@@ -1,4 +1,4 @@
-import React from "react";
+import { FC } from "react";
 import { DownloadButton } from "../components/DownloadButton";
 import "./DownloadPanel.css";
 
@@ -9,7 +9,7 @@ interface DownloadPanelProps {
   fileContents: string | Blob | ArrayBuffer | ArrayBufferView;
 }
 
-export const DownloadPanel: React.FC<DownloadPanelProps> = ({
+export const DownloadPanel: FC<DownloadPanelProps> = ({
   message,
   buttonLabel,
   fileName,