inspect-ai 0.3.58__py3-none-any.whl → 0.3.60__py3-none-any.whl

inspect_ai/_view/www/log-schema.json

@@ -1137,6 +1137,7 @@
             "logprobs": null,
             "top_logprobs": null,
             "parallel_tool_calls": null,
+            "internal_tools": null,
             "max_tool_output": null,
             "cache_prompt": null,
             "reasoning_effort": null
@@ -1516,7 +1517,8 @@
             "time",
             "message",
             "token",
-            "operator"
+            "operator",
+            "custom"
           ],
           "title": "Type",
           "type": "string"
@@ -2190,6 +2192,18 @@
           "default": null,
           "title": "Parallel Tool Calls"
         },
+        "internal_tools": {
+          "anyOf": [
+            {
+              "type": "boolean"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Internal Tools"
+        },
         "max_tool_output": {
           "anyOf": [
             {
@@ -2258,6 +2272,7 @@
         "logprobs",
         "top_logprobs",
         "parallel_tool_calls",
+        "internal_tools",
         "max_tool_output",
         "cache_prompt",
         "reasoning_effort"
@@ -2681,6 +2696,18 @@
         "output": {
           "$ref": "#/$defs/ModelOutput"
         },
+        "error": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Error"
+        },
         "cache": {
           "anyOf": [
             {
@@ -2719,6 +2746,7 @@
         "tool_choice",
         "config",
         "output",
+        "error",
         "cache",
         "call"
       ],
@@ -3066,7 +3094,8 @@
             "message",
             "time",
             "token",
-            "operator"
+            "operator",
+            "custom"
           ],
           "title": "Type",
           "type": "string"
@@ -4207,6 +4236,7 @@
           "best_of": null,
           "cache_prompt": null,
           "frequency_penalty": null,
+          "internal_tools": null,
           "logit_bias": null,
           "logprobs": null,
           "max_connections": null,

inspect_ai/_view/www/package.json

@@ -30,8 +30,10 @@
     "bootstrap": "^5.3.3",
     "bootstrap-icons": "^1.11.3",
     "clipboard": "^2.0.11",
+    "codemirror": "^6.0.1",
     "fast-json-patch": "^3.1.1",
     "fflate": "^0.8.2",
+    "filtrex": "^3.1.0",
     "htm": "^3.1.1",
     "json": "^11.0.0",
     "json5": "^2.2.3",

inspect_ai/_view/www/src/App.mjs

@@ -30,7 +30,7 @@ import { Sidebar } from "./sidebar/Sidebar.mjs";
 import { WorkSpace } from "./workspace/WorkSpace.mjs";
 import { FindBand } from "./components/FindBand.mjs";
 import { isVscode } from "./utils/Html.mjs";
-import { getVscodeApi } from "./utils/vscode.mjs";
+import { getVscodeApi } from "./utils/vscode";
 import { kDefaultSort } from "./constants.mjs";
 import {
   createEvalDescriptor,
@@ -38,7 +38,7 @@ import {
 } from "./samples/SamplesDescriptor.mjs";
 import { byEpoch, bySample, sortSamples } from "./samples/tools/SortFilter.mjs";
 import { resolveAttachments } from "./utils/attachments.mjs";
-import { filterFnForType } from "./samples/tools/filters.mjs";
+import { filterSamples } from "./samples/tools/filters.mjs";
 
 import {
   kEvalWorkspaceTabId,
@@ -51,7 +51,7 @@ import {
  * Renders the Main Application
  *
  * @param {Object} props - The parameters for the component.
- * @param {import("./api/Types.mjs").ClientAPI} props.api - The api that this view should use
+ * @param {import("./api/Types.ts").ClientAPI} props.api - The api that this view should use
  * @param {Object} [props.initialState] - Initial state for app (optional, used by VS Code extension)
  * @param {(state: Object) => void} [props.saveInitialState] - Save initial state for app (optional, used by VS Code extension)
  * @param {boolean} props.pollForLogs - Whether the application should poll for log changes
@@ -308,21 +308,19 @@ export function App({
 
   useEffect(() => {
     const samples = selectedLog?.contents?.sampleSummaries || [];
-    const filtered = samples.filter((sample) => {
+    const { result: prefiltered } = filterSamples(
+      evalDescriptor,
+      samples,
+      filter?.value,
+    );
+    const filtered = prefiltered.filter((sample) => {
       // Filter by epoch if specified
       if (epoch && epoch !== "all") {
         if (epoch !== sample.epoch + "") {
           return false;
         }
       }
-
-      // Apply the filter
-      const filterFn = filterFnForType(filter);
-      if (filterFn && filter.value) {
-        return filterFn(samplesDescriptor, sample, filter.value);
-      } else {
-        return true;
-      }
+      return true;
     });
 
     // Sort the samples
@@ -509,12 +507,12 @@ export function App({
    * Determines whether the workspace tab should display samples or info,
    * depending on the presence of samples and the log status.
    *
-   * @param {import("./api/Types.mjs").EvalSummary} log - The log object containing sample summaries and status.
+   * @param {import("./api/Types.ts").EvalSummary} log - The log object containing sample summaries and status.
    * @returns {void}
    */
   const resetWorkspace = useCallback(
     /**
-     * @param {import("./api/Types.mjs").EvalSummary} log
+     * @param {import("./api/Types.ts").EvalSummary} log
      */
     (log) => {
       // Reset the workspace tab
@@ -961,7 +959,7 @@ export function App({
 /**
  * Determines the default scorer for a log
  *
- * @param {import("./api/Types.mjs").EvalSummary} log - The log object containing sample summaries and status.
+ * @param {import("./api/Types.ts").EvalSummary} log - The log object containing sample summaries and status.
  * @returns {{name: string, scorer: string} | undefined} A scorer object with name and scorer properties, or undefined
  */
 const defaultScorer = (log) => {
@@ -983,7 +981,7 @@ const defaultScorer = (log) => {
 /**
  * Determines the default scorers for a log
  *
- * @param {import("./api/Types.mjs").EvalSummary} log - The log object containing sample summaries and status.
+ * @param {import("./api/Types.ts").EvalSummary} log - The log object containing sample summaries and status.
  * @returns {Array<{name: string, scorer: string}>} An array of scorer objects with name and scorer properties, or an empty array if no scorers are found.
  */
 const defaultScorers = (log) => {

inspect_ai/_view/www/src/Types.mjs

@@ -7,7 +7,7 @@
 /**
  * @typedef {Object} CurrentLog
  * @property {string} name
- * @property {import("./api/Types.mjs").EvalSummary} contents
+ * @property {import("./api/Types.ts").EvalSummary} contents
  */
 
 /**
@@ -25,7 +25,6 @@
 /**
  * @typedef {Object} ScoreFilter
  * @property {string} [value]
- * @property {string} [type]
  */
 
 /**

inspect_ai/_view/www/src/api/Types.ts

@@ -0,0 +1,133 @@
+import {
+  Version,
+  Status,
+  EvalSpec,
+  EvalPlan,
+  EvalResults,
+  EvalStats,
+  EvalError,
+  Input,
+  Target,
+  Scores1,
+  Type11,
+  EvalLog,
+  EvalSample,
+} from "../types/log";
+
+export interface EvalSummary {
+  version?: Version;
+  status?: Status;
+  eval: EvalSpec;
+  plan?: EvalPlan;
+  results?: EvalResults | null;
+  stats?: EvalStats;
+  error?: EvalError | null;
+  sampleSummaries: SampleSummary[];
+}
+
+export interface EvalLogHeader {
+  version?: Version;
+  status?: Status;
+  eval: EvalSpec;
+  plan?: EvalPlan;
+  results?: EvalResults;
+  stats?: EvalStats;
+  error?: EvalError;
+}
+
+export interface SampleSummary {
+  id: number | string;
+  epoch: number;
+  input: Input;
+  target: Target;
+  scores: Scores1;
+  error?: string;
+  limit?: Type11;
+}
+
+export interface BasicSampleData {
+  id: number | string;
+  epoch: number;
+  target: Target;
+  scores: Scores1;
+}
+
+export interface Capabilities {
+  downloadFiles: boolean;
+  webWorkers: boolean;
+}
+
+export interface LogViewAPI {
+  client_events: () => Promise<any[]>;
+  eval_logs: () => Promise<LogFiles | undefined>;
+  eval_log: (
+    log_file: string,
+    headerOnly?: number,
+    capabilities?: Capabilities,
+  ) => Promise<LogContents>;
+  eval_log_size: (log_file: string) => Promise<number>;
+  eval_log_bytes: (
+    log_file: string,
+    start: number,
+    end: number,
+  ) => Promise<Uint8Array>;
+  eval_log_headers: (log_files: string[]) => Promise<EvalLog[]>;
+  download_file: (
+    filename: string,
+    filecontents: string | Blob | ArrayBuffer | ArrayBufferView,
+  ) => Promise<void>;
+  open_log_file: (logFile: string, log_dir: string) => Promise<void>;
+}
+
+export interface ClientAPI {
+  client_events: () => Promise<string[]>;
+  get_log_paths: () => Promise<LogFiles>;
+  get_log_headers: (log_files: string[]) => Promise<EvalLog[]>;
+  get_log_summary: (log_file: string) => Promise<EvalSummary>;
+  get_log_sample: (
+    log_file: string,
+    id: string | number,
+    epoch: number,
+  ) => Promise<EvalSample | undefined>;
+  download_file: (
+    file_name: string,
+    file_contents: string | Blob | ArrayBuffer | ArrayBufferView,
+  ) => Promise<void>;
+  open_log_file: (log_file: string, log_dir: string) => Promise<void>;
+}
+
+export interface FetchResponse {
+  raw: string;
+  parsed: Record<string, any>;
+}
+
+export interface EvalHeader {
+  version?: Version;
+  status?: Status;
+  eval: EvalSpec;
+  plan?: EvalPlan;
+  results?: EvalResults | null;
+  stats?: EvalStats;
+  error?: EvalError | null;
+}
+
+export interface LogFiles {
+  files: LogFile[];
+  log_dir?: string;
+}
+
+export interface LogFile {
+  name: string;
+  task: string;
+  task_id: string;
+}
+
+export interface LogContents {
+  raw: string;
+  parsed: EvalLog;
+}
+
+export interface LogFilesFetchResponse {
+  raw: string;
+  parsed: Record<string, EvalHeader>;
+}

inspect_ai/_view/www/src/api/{api-browser.mjs → api-browser.ts}

@@ -1,6 +1,7 @@
-//@ts-check
-import { asyncJsonParse } from "../utils/Json.mjs";
-import { download_file } from "./api-shared.mjs";
+import { Capabilities } from "../Types.mjs";
+import { asyncJsonParse } from "../utils/json-worker";
+import { download_file } from "./api-shared";
+import { LogContents, LogViewAPI } from "./Types";
 
 const loaded_time = Date.now();
 let last_eval_time = 0;
@@ -18,25 +19,29 @@ async function eval_logs() {
   return logs.parsed;
 }
 
-async function eval_log(file, headerOnly) {
+async function eval_log(
+  file: string,
+  headerOnly?: number,
+  _capabilities?: Capabilities,
+): Promise<LogContents> {
   return await api(
     "GET",
     `/api/logs/${encodeURIComponent(file)}?header-only=${headerOnly}`,
   );
 }
 
-async function eval_log_size(file) {
+async function eval_log_size(file: string): Promise<number> {
   return (await api("GET", `/api/log-size/${encodeURIComponent(file)}`)).parsed;
 }
 
-async function eval_log_bytes(file, start, end) {
+async function eval_log_bytes(file: string, start: number, end: number) {
   return await api_bytes(
     "GET",
     `/api/log-bytes/${encodeURIComponent(file)}?start=${start}&end=${end}`,
   );
 }
 
-async function eval_log_headers(files) {
+async function eval_log_headers(files: string[]) {
   const params = new URLSearchParams();
   for (const file of files) {
     params.append("file", file);
@@ -44,9 +49,13 @@ async function eval_log_headers(files) {
   return (await api("GET", `/api/log-headers?${params.toString()}`)).parsed;
 }
 
-async function api(method, path, body) {
+async function api(
+  method: "GET" | "POST" | "PUT" | "DELETE",
+  path: string,
+  body?: string,
+) {
   // build headers
-  const headers = {
+  const headers: HeadersInit = {
     Accept: "application/json",
     Pragma: "no-cache",
     Expires: "0",
@@ -73,9 +82,12 @@ async function api(method, path, body) {
   }
 }
 
-async function api_bytes(method, path) {
+async function api_bytes(
+  method: "GET" | "POST" | "PUT" | "DELETE",
+  path: string,
+) {
   // build headers
-  const headers = {
+  const headers: HeadersInit = {
     Accept: "application/octet-stream",
     Pragma: "no-cache",
     Expires: "0",
@@ -100,8 +112,7 @@ async function open_log_file() {
   // No op
 }
 
-/** @type {import("./Types.mjs").LogViewAPI} */
-export default {
+const browserApi: LogViewAPI = {
   client_events,
   eval_logs,
   eval_log,
@@ -111,3 +122,4 @@ export default {
   download_file,
   open_log_file,
 };
+export default browserApi;

inspect_ai/_view/www/src/api/api-http.ts

@@ -0,0 +1,219 @@
+//@ts-check
+import { asyncJsonParse } from "../utils/json-worker";
+import { download_file, encodePathParts } from "./api-shared";
+import { fetchRange, fetchSize } from "../utils/remoteZipFile.mjs";
+import {
+  Capabilities,
+  LogContents,
+  LogFiles,
+  LogFilesFetchResponse,
+  LogViewAPI,
+} from "./Types";
+import { EvalLog } from "../types/log";
+
+interface LogInfo {
+  log_dir?: string;
+  log_file?: string;
+}
+
+/**
+ * This provides an API implementation that will serve a single
+ * file using an http parameter, designed to be deployed
+ * to a webserver without inspect or the ability to enumerate log
+ * files
+ */
+export default function simpleHttpApi(
+  log_dir?: string,
+  log_file?: string,
+): LogViewAPI {
+  const resolved_log_dir = log_dir?.replace(" ", "+");
+  const resolved_log_path = log_file ? log_file.replace(" ", "+") : undefined;
+  return simpleHttpAPI({
+    log_file: resolved_log_path,
+    log_dir: resolved_log_dir,
+  });
+}
+
+/**
+ * Fetches a file from the specified URL and parses its content.
+ */
+function simpleHttpAPI(logInfo: LogInfo): LogViewAPI {
+  const log_file = logInfo.log_file;
+  const log_dir = logInfo.log_dir;
+
+  async function open_log_file() {
+    // No op
+  }
+  return {
+    client_events: async () => {
+      // There are no client events in the case of serving via
+      // http
+      return Promise.resolve([]);
+    },
+    eval_logs: async (): Promise<LogFiles | undefined> => {
+      // First check based upon the log dir
+      if (log_dir) {
+        const headers = await fetchLogHeaders(log_dir);
+        if (headers) {
+          const logRecord = headers.parsed;
+          const logs = Object.keys(logRecord).map((key) => {
+            return {
+              name: joinURI(log_dir, key),
+              task: logRecord[key].eval.task,
+              task_id: logRecord[key].eval.task_id,
+            };
+          });
+          return Promise.resolve({
+            files: logs,
+            log_dir,
+          });
+        }
+      }
+
+      return undefined;
+    },
+    eval_log: async (
+      log_file: string,
+      _headerOnly?: number,
+      _capabilities?: Capabilities,
+    ) => {
+      const response = await fetchLogFile(log_file);
+      if (response) {
+        return response;
+      } else {
+        throw new Error(`"Unable to load eval log ${log_file}`);
+      }
+    },
+    eval_log_size: async (log_file: string) => {
+      return await fetchSize(log_file);
+    },
+    eval_log_bytes: async (log_file: string, start: number, end: number) => {
+      return await fetchRange(log_file, start, end);
+    },
+    eval_log_headers: async (files: string[]) => {
+      if (files.length === 0) {
+        return [];
+      }
+
+      if (log_dir) {
+        const headers = await fetchLogHeaders(log_dir);
+        if (headers) {
+          const keys = Object.keys(headers.parsed);
+          const result: EvalLog[] = [];
+          files.forEach((file) => {
+            const fileKey = keys.find((key) => {
+              return file.endsWith(key);
+            });
+            if (fileKey) {
+              result.push(headers.parsed[fileKey]);
+            }
+          });
+          return result;
+        }
+      }
+
+      // No log.json could be found, and there isn't a log file,
+      throw new Error(
+        `Failed to load a manifest files using the directory: ${log_dir}. Please be sure you have deployed a manifest file (logs.json).`,
+      );
+    },
+    download_file,
+    open_log_file,
+  };
+}
+
+/**
+ * Fetches a file from the specified URL and parses its content.
+ */
+async function fetchFile<T>(
+  url: string,
+  parse: (text: string) => Promise<T>,
+  handleError?: (response: Response) => boolean,
+): Promise<T | undefined> {
+  const safe_url = encodePathParts(url);
+  const response = await fetch(`${safe_url}`, { method: "GET" });
+  if (response.ok) {
+    const text = await response.text();
+    return await parse(text);
+  } else if (response.status !== 200) {
+    if (handleError && handleError(response)) {
+      return undefined;
+    }
+    const message = (await response.text()) || response.statusText;
+    const error = new Error(`${response.status}: ${message})`);
+    throw error;
+  } else {
+    throw new Error(`${response.status} - ${response.statusText} `);
+  }
+}
+
+/**
+ * Fetches a log file and parses its content, updating the log structure if necessary.
+ */
+const fetchLogFile = async (file: string): Promise<LogContents | undefined> => {
+  return fetchFile<LogContents>(file, async (text): Promise<LogContents> => {
+    const log = (await asyncJsonParse(text)) as EvalLog;
+    if (log.version === 1) {
+      if (log.results) {
+        const untypedLog = log as any;
+        log.results.scores = [];
+        untypedLog.results.scorer.scorer = untypedLog.results.scorer.name;
+        log.results.scores.push(untypedLog.results.scorer);
+        delete untypedLog.results.scorer;
+        log.results.scores[0].metrics = untypedLog.results.metrics;
+        delete untypedLog.results.metrics;
+
+        // migrate samples
+        const scorerName = log.results.scores[0].name;
+        log.samples?.forEach((sample) => {
+          const untypedSample = sample as any;
+          sample.scores = { [scorerName]: untypedSample.score };
+          delete untypedSample.score;
+        });
+      }
+    }
+    return {
+      raw: text,
+      parsed: log,
+    };
+  });
+};
+
+/**
+ * Fetches a log file and parses its content, updating the log structure if necessary.
+ */
+const fetchLogHeaders = async (
+  log_dir: string,
+): Promise<LogFilesFetchResponse | undefined> => {
+  const logs = await fetchFile<LogFilesFetchResponse>(
+    log_dir + "/logs.json",
+    async (text) => {
+      const parsed = await asyncJsonParse(text);
+      return {
+        raw: text,
+        parsed,
+      };
+    },
+    (response) => {
+      if (response.status === 404) {
+        // Couldn't find a header file
+        return true;
+      } else {
+        return false;
+      }
+    },
+  );
+  return logs;
+};
+
+/**
+ * Joins multiple URI segments into a single URI string.
+ *
+ * This function removes any leading or trailing slashes from each segment
+ * and then joins them with a single slash (`/`).
+ */
+function joinURI(...segments: string[]): string {
+  return segments
+    .map((segment) => segment.replace(/(^\/+|\/+$)/g, "")) // Remove leading/trailing slashes from each segment
+    .join("/");
+}

inspect_ai/_view/www/src/api/api-shared.ts

@@ -0,0 +1,47 @@
+/**
+ * Downloads the provided content as a file using the browser's DOM API
+ */
+export async function download_file(
+  filename: string,
+  filecontents: string | Blob | ArrayBuffer | ArrayBufferView,
+): Promise<void> {
+  const blob = new Blob([filecontents], { type: "text/plain" });
+  const link = document.createElement("a");
+  link.href = URL.createObjectURL(blob);
+  link.download = filename;
+  document.body.appendChild(link);
+  link.click();
+  document.body.removeChild(link);
+}
+
+/**
+ * Encodes the path segments of a URL or relative path to ensure special characters
+ * (like `+`, spaces, etc.) are properly encoded without affecting legal characters like `/`.
+ *
+ * This function will encode file names and path portions of both absolute URLs and
+ * relative paths. It ensures that components of a full URL, such as the protocol and
+ * query parameters, remain intact, while only encoding the path.
+ */
+export function encodePathParts(url: string): string {
+  if (!url) return url; // Handle empty strings
+
+  try {
+    // Parse a full Uri
+    const fullUrl = new URL(url);
+    fullUrl.pathname = fullUrl.pathname
+      .split("/")
+      .map((segment) =>
+        segment ? encodeURIComponent(decodeURIComponent(segment)) : "",
+      )
+      .join("/");
+    return fullUrl.toString();
+  } catch {
+    // This is a relative path that isn't parseable as Uri
+    return url
+      .split("/")
+      .map((segment) =>
+        segment ? encodeURIComponent(decodeURIComponent(segment)) : "",
+      )
+      .join("/");
+  }
+}