inspect-ai 0.3.59__py3-none-any.whl → 0.3.60__py3-none-any.whl

inspect_ai/_view/www/src/App.mjs

@@ -30,7 +30,7 @@ import { Sidebar } from "./sidebar/Sidebar.mjs";
 import { WorkSpace } from "./workspace/WorkSpace.mjs";
 import { FindBand } from "./components/FindBand.mjs";
 import { isVscode } from "./utils/Html.mjs";
-import { getVscodeApi } from "./utils/vscode.mjs";
+import { getVscodeApi } from "./utils/vscode";
 import { kDefaultSort } from "./constants.mjs";
 import {
   createEvalDescriptor,
@@ -51,7 +51,7 @@ import {
  * Renders the Main Application
  *
  * @param {Object} props - The parameters for the component.
- * @param {import("./api/Types.mjs").ClientAPI} props.api - The api that this view should use
+ * @param {import("./api/Types.ts").ClientAPI} props.api - The api that this view should use
  * @param {Object} [props.initialState] - Initial state for app (optional, used by VS Code extension)
  * @param {(state: Object) => void} [props.saveInitialState] - Save initial state for app (optional, used by VS Code extension)
  * @param {boolean} props.pollForLogs - Whether the application should poll for log changes
@@ -507,12 +507,12 @@ export function App({
    * Determines whether the workspace tab should display samples or info,
    * depending on the presence of samples and the log status.
    *
-   * @param {import("./api/Types.mjs").EvalSummary} log - The log object containing sample summaries and status.
+   * @param {import("./api/Types.ts").EvalSummary} log - The log object containing sample summaries and status.
    * @returns {void}
    */
   const resetWorkspace = useCallback(
     /**
-     * @param {import("./api/Types.mjs").EvalSummary} log
+     * @param {import("./api/Types.ts").EvalSummary} log
      */
     (log) => {
       // Reset the workspace tab
@@ -959,7 +959,7 @@ export function App({
 /**
  * Determines the default scorer for a log
  *
- * @param {import("./api/Types.mjs").EvalSummary} log - The log object containing sample summaries and status.
+ * @param {import("./api/Types.ts").EvalSummary} log - The log object containing sample summaries and status.
  * @returns {{name: string, scorer: string} | undefined} A scorer object with name and scorer properties, or undefined
  */
 const defaultScorer = (log) => {
@@ -981,7 +981,7 @@ const defaultScorer = (log) => {
 /**
  * Determines the default scorers for a log
  *
- * @param {import("./api/Types.mjs").EvalSummary} log - The log object containing sample summaries and status.
+ * @param {import("./api/Types.ts").EvalSummary} log - The log object containing sample summaries and status.
  * @returns {Array<{name: string, scorer: string}>} An array of scorer objects with name and scorer properties, or an empty array if no scorers are found.
  */
 const defaultScorers = (log) => {

inspect_ai/_view/www/src/Types.mjs

@@ -7,7 +7,7 @@
 /**
  * @typedef {Object} CurrentLog
  * @property {string} name
- * @property {import("./api/Types.mjs").EvalSummary} contents
+ * @property {import("./api/Types.ts").EvalSummary} contents
  */
 
 /**

inspect_ai/_view/www/src/api/Types.ts

@@ -0,0 +1,133 @@
+import {
+  Version,
+  Status,
+  EvalSpec,
+  EvalPlan,
+  EvalResults,
+  EvalStats,
+  EvalError,
+  Input,
+  Target,
+  Scores1,
+  Type11,
+  EvalLog,
+  EvalSample,
+} from "../types/log";
+
+export interface EvalSummary {
+  version?: Version;
+  status?: Status;
+  eval: EvalSpec;
+  plan?: EvalPlan;
+  results?: EvalResults | null;
+  stats?: EvalStats;
+  error?: EvalError | null;
+  sampleSummaries: SampleSummary[];
+}
+
+export interface EvalLogHeader {
+  version?: Version;
+  status?: Status;
+  eval: EvalSpec;
+  plan?: EvalPlan;
+  results?: EvalResults;
+  stats?: EvalStats;
+  error?: EvalError;
+}
+
+export interface SampleSummary {
+  id: number | string;
+  epoch: number;
+  input: Input;
+  target: Target;
+  scores: Scores1;
+  error?: string;
+  limit?: Type11;
+}
+
+export interface BasicSampleData {
+  id: number | string;
+  epoch: number;
+  target: Target;
+  scores: Scores1;
+}
+
+export interface Capabilities {
+  downloadFiles: boolean;
+  webWorkers: boolean;
+}
+
+export interface LogViewAPI {
+  client_events: () => Promise<any[]>;
+  eval_logs: () => Promise<LogFiles | undefined>;
+  eval_log: (
+    log_file: string,
+    headerOnly?: number,
+    capabilities?: Capabilities,
+  ) => Promise<LogContents>;
+  eval_log_size: (log_file: string) => Promise<number>;
+  eval_log_bytes: (
+    log_file: string,
+    start: number,
+    end: number,
+  ) => Promise<Uint8Array>;
+  eval_log_headers: (log_files: string[]) => Promise<EvalLog[]>;
+  download_file: (
+    filename: string,
+    filecontents: string | Blob | ArrayBuffer | ArrayBufferView,
+  ) => Promise<void>;
+  open_log_file: (logFile: string, log_dir: string) => Promise<void>;
+}
+
+export interface ClientAPI {
+  client_events: () => Promise<string[]>;
+  get_log_paths: () => Promise<LogFiles>;
+  get_log_headers: (log_files: string[]) => Promise<EvalLog[]>;
+  get_log_summary: (log_file: string) => Promise<EvalSummary>;
+  get_log_sample: (
+    log_file: string,
+    id: string | number,
+    epoch: number,
+  ) => Promise<EvalSample | undefined>;
+  download_file: (
+    file_name: string,
+    file_contents: string | Blob | ArrayBuffer | ArrayBufferView,
+  ) => Promise<void>;
+  open_log_file: (log_file: string, log_dir: string) => Promise<void>;
+}
+
+export interface FetchResponse {
+  raw: string;
+  parsed: Record<string, any>;
+}
+
+export interface EvalHeader {
+  version?: Version;
+  status?: Status;
+  eval: EvalSpec;
+  plan?: EvalPlan;
+  results?: EvalResults | null;
+  stats?: EvalStats;
+  error?: EvalError | null;
+}
+
+export interface LogFiles {
+  files: LogFile[];
+  log_dir?: string;
+}
+
+export interface LogFile {
+  name: string;
+  task: string;
+  task_id: string;
+}
+
+export interface LogContents {
+  raw: string;
+  parsed: EvalLog;
+}
+
+export interface LogFilesFetchResponse {
+  raw: string;
+  parsed: Record<string, EvalHeader>;
+}

inspect_ai/_view/www/src/api/{api-browser.mjs → api-browser.ts}

@@ -1,6 +1,7 @@
-//@ts-check
-import { asyncJsonParse } from "../utils/Json.mjs";
-import { download_file } from "./api-shared.mjs";
+import { Capabilities } from "../Types.mjs";
+import { asyncJsonParse } from "../utils/json-worker";
+import { download_file } from "./api-shared";
+import { LogContents, LogViewAPI } from "./Types";
 
 const loaded_time = Date.now();
 let last_eval_time = 0;
@@ -18,25 +19,29 @@ async function eval_logs() {
   return logs.parsed;
 }
 
-async function eval_log(file, headerOnly) {
+async function eval_log(
+  file: string,
+  headerOnly?: number,
+  _capabilities?: Capabilities,
+): Promise<LogContents> {
   return await api(
     "GET",
     `/api/logs/${encodeURIComponent(file)}?header-only=${headerOnly}`,
   );
 }
 
-async function eval_log_size(file) {
+async function eval_log_size(file: string): Promise<number> {
   return (await api("GET", `/api/log-size/${encodeURIComponent(file)}`)).parsed;
 }
 
-async function eval_log_bytes(file, start, end) {
+async function eval_log_bytes(file: string, start: number, end: number) {
   return await api_bytes(
     "GET",
     `/api/log-bytes/${encodeURIComponent(file)}?start=${start}&end=${end}`,
   );
 }
 
-async function eval_log_headers(files) {
+async function eval_log_headers(files: string[]) {
   const params = new URLSearchParams();
   for (const file of files) {
     params.append("file", file);
@@ -44,9 +49,13 @@ async function eval_log_headers(files) {
   return (await api("GET", `/api/log-headers?${params.toString()}`)).parsed;
 }
 
-async function api(method, path, body) {
+async function api(
+  method: "GET" | "POST" | "PUT" | "DELETE",
+  path: string,
+  body?: string,
+) {
   // build headers
-  const headers = {
+  const headers: HeadersInit = {
     Accept: "application/json",
     Pragma: "no-cache",
     Expires: "0",
@@ -73,9 +82,12 @@ async function api(method, path, body) {
   }
 }
 
-async function api_bytes(method, path) {
+async function api_bytes(
+  method: "GET" | "POST" | "PUT" | "DELETE",
+  path: string,
+) {
   // build headers
-  const headers = {
+  const headers: HeadersInit = {
     Accept: "application/octet-stream",
     Pragma: "no-cache",
     Expires: "0",
@@ -100,8 +112,7 @@ async function open_log_file() {
   // No op
 }
 
-/** @type {import("./Types.mjs").LogViewAPI} */
-export default {
+const browserApi: LogViewAPI = {
   client_events,
   eval_logs,
   eval_log,
@@ -111,3 +122,4 @@ export default {
   download_file,
   open_log_file,
 };
+export default browserApi;

inspect_ai/_view/www/src/api/api-http.ts

@@ -0,0 +1,219 @@
+//@ts-check
+import { asyncJsonParse } from "../utils/json-worker";
+import { download_file, encodePathParts } from "./api-shared";
+import { fetchRange, fetchSize } from "../utils/remoteZipFile.mjs";
+import {
+  Capabilities,
+  LogContents,
+  LogFiles,
+  LogFilesFetchResponse,
+  LogViewAPI,
+} from "./Types";
+import { EvalLog } from "../types/log";
+
+interface LogInfo {
+  log_dir?: string;
+  log_file?: string;
+}
+
+/**
+ * This provides an API implementation that will serve a single
+ * file using an http parameter, designed to be deployed
+ * to a webserver without inspect or the ability to enumerate log
+ * files
+ */
+export default function simpleHttpApi(
+  log_dir?: string,
+  log_file?: string,
+): LogViewAPI {
+  const resolved_log_dir = log_dir?.replace(" ", "+");
+  const resolved_log_path = log_file ? log_file.replace(" ", "+") : undefined;
+  return simpleHttpAPI({
+    log_file: resolved_log_path,
+    log_dir: resolved_log_dir,
+  });
+}
+
+/**
+ * Fetches a file from the specified URL and parses its content.
+ */
+function simpleHttpAPI(logInfo: LogInfo): LogViewAPI {
+  const log_file = logInfo.log_file;
+  const log_dir = logInfo.log_dir;
+
+  async function open_log_file() {
+    // No op
+  }
+  return {
+    client_events: async () => {
+      // There are no client events in the case of serving via
+      // http
+      return Promise.resolve([]);
+    },
+    eval_logs: async (): Promise<LogFiles | undefined> => {
+      // First check based upon the log dir
+      if (log_dir) {
+        const headers = await fetchLogHeaders(log_dir);
+        if (headers) {
+          const logRecord = headers.parsed;
+          const logs = Object.keys(logRecord).map((key) => {
+            return {
+              name: joinURI(log_dir, key),
+              task: logRecord[key].eval.task,
+              task_id: logRecord[key].eval.task_id,
+            };
+          });
+          return Promise.resolve({
+            files: logs,
+            log_dir,
+          });
+        }
+      }
+
+      return undefined;
+    },
+    eval_log: async (
+      log_file: string,
+      _headerOnly?: number,
+      _capabilities?: Capabilities,
+    ) => {
+      const response = await fetchLogFile(log_file);
+      if (response) {
+        return response;
+      } else {
+        throw new Error(`"Unable to load eval log ${log_file}`);
+      }
+    },
+    eval_log_size: async (log_file: string) => {
+      return await fetchSize(log_file);
+    },
+    eval_log_bytes: async (log_file: string, start: number, end: number) => {
+      return await fetchRange(log_file, start, end);
+    },
+    eval_log_headers: async (files: string[]) => {
+      if (files.length === 0) {
+        return [];
+      }
+
+      if (log_dir) {
+        const headers = await fetchLogHeaders(log_dir);
+        if (headers) {
+          const keys = Object.keys(headers.parsed);
+          const result: EvalLog[] = [];
+          files.forEach((file) => {
+            const fileKey = keys.find((key) => {
+              return file.endsWith(key);
+            });
+            if (fileKey) {
+              result.push(headers.parsed[fileKey]);
+            }
+          });
+          return result;
+        }
+      }
+
+      // No log.json could be found, and there isn't a log file,
+      throw new Error(
+        `Failed to load a manifest files using the directory: ${log_dir}. Please be sure you have deployed a manifest file (logs.json).`,
+      );
+    },
+    download_file,
+    open_log_file,
+  };
+}
+
+/**
+ * Fetches a file from the specified URL and parses its content.
+ */
+async function fetchFile<T>(
+  url: string,
+  parse: (text: string) => Promise<T>,
+  handleError?: (response: Response) => boolean,
+): Promise<T | undefined> {
+  const safe_url = encodePathParts(url);
+  const response = await fetch(`${safe_url}`, { method: "GET" });
+  if (response.ok) {
+    const text = await response.text();
+    return await parse(text);
+  } else if (response.status !== 200) {
+    if (handleError && handleError(response)) {
+      return undefined;
+    }
+    const message = (await response.text()) || response.statusText;
+    const error = new Error(`${response.status}: ${message})`);
+    throw error;
+  } else {
+    throw new Error(`${response.status} - ${response.statusText} `);
+  }
+}
+
+/**
+ * Fetches a log file and parses its content, updating the log structure if necessary.
+ */
+const fetchLogFile = async (file: string): Promise<LogContents | undefined> => {
+  return fetchFile<LogContents>(file, async (text): Promise<LogContents> => {
+    const log = (await asyncJsonParse(text)) as EvalLog;
+    if (log.version === 1) {
+      if (log.results) {
+        const untypedLog = log as any;
+        log.results.scores = [];
+        untypedLog.results.scorer.scorer = untypedLog.results.scorer.name;
+        log.results.scores.push(untypedLog.results.scorer);
+        delete untypedLog.results.scorer;
+        log.results.scores[0].metrics = untypedLog.results.metrics;
+        delete untypedLog.results.metrics;
+
+        // migrate samples
+        const scorerName = log.results.scores[0].name;
+        log.samples?.forEach((sample) => {
+          const untypedSample = sample as any;
+          sample.scores = { [scorerName]: untypedSample.score };
+          delete untypedSample.score;
+        });
+      }
+    }
+    return {
+      raw: text,
+      parsed: log,
+    };
+  });
+};
+
+/**
+ * Fetches a log file and parses its content, updating the log structure if necessary.
+ */
+const fetchLogHeaders = async (
+  log_dir: string,
+): Promise<LogFilesFetchResponse | undefined> => {
+  const logs = await fetchFile<LogFilesFetchResponse>(
+    log_dir + "/logs.json",
+    async (text) => {
+      const parsed = await asyncJsonParse(text);
+      return {
+        raw: text,
+        parsed,
+      };
+    },
+    (response) => {
+      if (response.status === 404) {
+        // Couldn't find a header file
+        return true;
+      } else {
+        return false;
+      }
+    },
+  );
+  return logs;
+};
+
+/**
+ * Joins multiple URI segments into a single URI string.
+ *
+ * This function removes any leading or trailing slashes from each segment
+ * and then joins them with a single slash (`/`).
+ */
+function joinURI(...segments: string[]): string {
+  return segments
+    .map((segment) => segment.replace(/(^\/+|\/+$)/g, "")) // Remove leading/trailing slashes from each segment
+    .join("/");
+}

inspect_ai/_view/www/src/api/api-shared.ts

@@ -0,0 +1,47 @@
+/**
+ * Downloads the provided content as a file using the browser's DOM API
+ */
+export async function download_file(
+  filename: string,
+  filecontents: string | Blob | ArrayBuffer | ArrayBufferView,
+): Promise<void> {
+  const blob = new Blob([filecontents], { type: "text/plain" });
+  const link = document.createElement("a");
+  link.href = URL.createObjectURL(blob);
+  link.download = filename;
+  document.body.appendChild(link);
+  link.click();
+  document.body.removeChild(link);
+}
+
+/**
+ * Encodes the path segments of a URL or relative path to ensure special characters
+ * (like `+`, spaces, etc.) are properly encoded without affecting legal characters like `/`.
+ *
+ * This function will encode file names and path portions of both absolute URLs and
+ * relative paths. It ensures that components of a full URL, such as the protocol and
+ * query parameters, remain intact, while only encoding the path.
+ */
+export function encodePathParts(url: string): string {
+  if (!url) return url; // Handle empty strings
+
+  try {
+    // Parse a full Uri
+    const fullUrl = new URL(url);
+    fullUrl.pathname = fullUrl.pathname
+      .split("/")
+      .map((segment) =>
+        segment ? encodeURIComponent(decodeURIComponent(segment)) : "",
+      )
+      .join("/");
+    return fullUrl.toString();
+  } catch {
+    // This is a relative path that isn't parseable as Uri
+    return url
+      .split("/")
+      .map((segment) =>
+        segment ? encodeURIComponent(decodeURIComponent(segment)) : "",
+      )
+      .join("/");
+  }
+}

inspect_ai/_view/www/src/api/{api-vscode.mjs → api-vscode.ts}

@@ -1,7 +1,4 @@
-//@ts-check
-
-import { asyncJsonParse } from "../utils/Json.mjs";
-// @ts-ignore
+import { asyncJsonParse } from "../utils/json-worker";
 import JSON5 from "json5";
 
 import {
@@ -11,8 +8,9 @@ import {
   kMethodEvalLogSize,
   kMethodEvalLogBytes,
   kMethodEvalLogHeaders,
-} from "./jsonrpc.mjs";
-import { getVscodeApi } from "../utils/vscode.mjs";
+} from "./jsonrpc";
+import { getVscodeApi } from "../utils/vscode";
+import { Capabilities, LogContents, LogViewAPI } from "./Types";
 
 const vscodeClient = webViewJsonRpcClient(getVscodeApi());
 
@@ -38,8 +36,12 @@ async function eval_logs() {
   }
 }
 
-async function eval_log(file, headerOnly, capabilities) {
-  const response = await vscodeClient(kMethodEvalLog, [file, headerOnly]);
+async function eval_log(
+  log_file: string,
+  headerOnly?: number,
+  capabilities?: Capabilities,
+): Promise<LogContents> {
+  const response = await vscodeClient(kMethodEvalLog, [log_file, headerOnly]);
   if (response) {
     let json;
     if (capabilities?.webWorkers) {
@@ -52,19 +54,19 @@ async function eval_log(file, headerOnly, capabilities) {
       raw: response,
     };
   } else {
-    return undefined;
+    throw new Error(`Unable to load eval log ${log_file}.`);
   }
 }
 
-async function eval_log_size(file) {
-  return await vscodeClient(kMethodEvalLogSize, [file]);
+async function eval_log_size(log_file: string) {
+  return await vscodeClient(kMethodEvalLogSize, [log_file]);
 }
 
-async function eval_log_bytes(file, start, end) {
-  return await vscodeClient(kMethodEvalLogBytes, [file, start, end]);
+async function eval_log_bytes(log_file: string, start: number, end: number) {
+  return await vscodeClient(kMethodEvalLogBytes, [log_file, start, end]);
 }
 
-async function eval_log_headers(files) {
+async function eval_log_headers(files: string[]) {
   const response = await vscodeClient(kMethodEvalLogHeaders, [files]);
   if (response) {
     return JSON5.parse(response);
@@ -77,17 +79,16 @@ async function download_file() {
   throw Error("Downloading files is not supported in VS Code");
 }
 
-async function open_log_file(url, log_dir) {
+async function open_log_file(log_file: string, log_dir: string) {
   const msg = {
     type: "displayLogFile",
-    url: url,
+    url: log_file,
     log_dir: log_dir,
   };
-  getVscodeApi().postMessage(msg);
+  getVscodeApi()?.postMessage(msg);
 }
 
-/** @type {import("./Types.mjs").LogViewAPI} */
-export default {
+const api: LogViewAPI = {
   client_events,
   eval_logs,
   eval_log,
@@ -97,3 +98,5 @@ export default {
   download_file,
   open_log_file,
 };
+
+export default api;