inspect-ai 0.3.103__py3-none-any.whl → 0.3.105__py3-none-any.whl

inspect_ai/_view/www/src/app/samples/sample-tools/sample-filter/completions.ts

@@ -3,8 +3,10 @@ import {
   CompletionContext,
   CompletionResult,
   CompletionSection,
+  startCompletion,
 } from "@codemirror/autocomplete";
 import { EditorView } from "codemirror";
+import { SampleSummary } from "../../../../client/api/types";
 import {
   kScoreTypeBoolean,
   kScoreTypeCategorical,
@@ -15,6 +17,8 @@ import {
 import { SampleFilterItem } from "../filters";
 import {
   KEYWORDS,
+  kSampleIdVariable,
+  kSampleMetadataVariable,
   MATH_FUNCTIONS,
   SAMPLE_FUNCTIONS,
   SAMPLE_VARIABLES,
@@ -53,6 +57,34 @@ const applyWithCall = (
   });
 };
 
+const applyWithDot = (
+  view: EditorView,
+  completion: Completion,
+  from: number,
+  to: number,
+): void => {
+  view.dispatch({
+    changes: { from, to, insert: `${completion.label}.` },
+    selection: { anchor: from + completion.label.length + 1 },
+  });
+  // trigger completion
+  setTimeout(() => startCompletion(view), 0);
+};
+
+const applyWithSpace = (
+  view: EditorView,
+  completion: Completion,
+  from: number,
+  to: number,
+): void => {
+  view.dispatch({
+    changes: { from, to, insert: `${completion.label} ` },
+    selection: { anchor: from + completion.label.length + 1 },
+  });
+  // trigger completion
+  setTimeout(() => startCompletion(view), 0);
+};
+
 const makeKeywordCompletion = (k: string): Completion => ({
   label: k,
   type: "keyword",
@@ -88,6 +120,12 @@ const makeSampleVariableCompletion = ([label, info]: [
   label,
   type: "variable",
   info,
+  apply:
+    label === kSampleMetadataVariable
+      ? applyWithDot
+      : label === kSampleIdVariable
+        ? applyWithSpace
+        : undefined,
   boost: 10,
 });
 
@@ -120,6 +158,210 @@ const getMemberScoreItems = (
 ): SampleFilterItem[] =>
   filterItems.filter((item) => item?.qualifiedName?.startsWith(`${scorer}.`));
 
+const getSampleIds = (samples: SampleSummary[]): Set<string | number> => {
+  const ids = new Set<string | number>();
+  for (const sample of samples) {
+    ids.add(sample.id);
+  }
+  return ids;
+};
+
+const getMetadataPropertyValues = (
+  samples: SampleSummary[],
+  propertyPath: string,
+): Set<any> => {
+  const values = new Set<any>();
+  for (const sample of samples) {
+    if (sample.metadata) {
+      const value = getNestedProperty(sample.metadata, propertyPath);
+      if (value !== undefined && value !== null) {
+        values.add(value);
+      }
+    }
+  }
+  return values;
+};
+
+const getNestedProperty = (obj: any, path: string): any => {
+  const keys = path.split(".");
+  let current = obj;
+  for (const key of keys) {
+    if (current && typeof current === "object" && key in current) {
+      current = current[key];
+    } else {
+      return undefined;
+    }
+  }
+  return current;
+};
+
+const buildMetadataPath = (
+  tokens: Token[],
+  currentTokenIndex: number,
+): string | null => {
+  // Walk backwards to build the metadata path
+  // For "metadata." return ""
+  // For "metadata.config." return "config"
+  // For "metadata.config.timeout." return "config.timeout"
+
+  const parts: string[] = [];
+
+  // Start after the first dot
+  let index = 2;
+
+  // Look for the metadata root by walking backwards
+  while (index <= currentTokenIndex) {
+    const token = tokens[currentTokenIndex - index];
+
+    if (token?.text === kSampleMetadataVariable) {
+      // Found metadata root, return the path
+      return parts.reverse().join(".");
+    } else if (token?.type === "variable") {
+      // Found a variable token, add to path
+      parts.push(token.text);
+      // Skip the expected dot
+      index++;
+      if (tokens[currentTokenIndex - index]?.text === ".") {
+        // Move past the dot
+        index++;
+      } else {
+        // No dot, not a valid path
+        break;
+      }
+    } else {
+      // Hit non-variable, non-metadata token
+      break;
+    }
+  }
+
+  // Didn't find metadata root
+  return null;
+};
+
+const getMetadataKeysForPath = (
+  samples: SampleSummary[],
+  parentPath: string,
+): Set<string> => {
+  const keys = new Set<string>();
+  for (const sample of samples) {
+    if (sample.metadata) {
+      const parentObj = parentPath
+        ? getNestedProperty(sample.metadata, parentPath)
+        : sample.metadata;
+      if (
+        parentObj &&
+        typeof parentObj === "object" &&
+        !Array.isArray(parentObj)
+      ) {
+        for (const key of Object.keys(parentObj)) {
+          keys.add(key);
+        }
+      }
+    }
+  }
+  return keys;
+};
+
+const buildMetadataPropertyPath = (
+  tokens: Token[],
+  currentTokenIndex: number,
+): string | null => {
+  // Walk backwards to build the full metadata property path
+  // e.g., for "metadata.difficulty ==" we want to return "difficulty"
+  // e.g., for "metadata.config.timeout ==" we want to return "config.timeout"
+  const parts: string[] = [];
+
+  // Start after the dot
+  let index = 2;
+
+  // Collect the property path by walking backwards
+  while (index <= currentTokenIndex) {
+    const token = tokens[currentTokenIndex - index];
+    if (!token) break;
+
+    if (token.type === "variable") {
+      if (token.text === kSampleMetadataVariable) {
+        // Found the metadata root, return the path
+        return parts.reverse().join(".");
+      } else {
+        parts.push(token.text);
+      }
+    } else if (token.text !== ".") {
+      // Hit a non-dot, non-variable token, not a metadata path
+      break;
+    }
+    index++;
+  }
+
+  return null;
+};
+
+const isMetadataProperty = (
+  tokens: Token[],
+  currentTokenIndex: number,
+): boolean => {
+  // Check if the current variable is part of a metadata property access
+  // e.g., for "metadata.difficulty" return true
+
+  // For metadata.difficulty, tokens are: [metadata, ., difficulty]
+  // currentTokenIndex points after difficulty, so prevToken(1) = difficulty
+  // We need to check if we can trace back to metadata
+
+  // Start by looking at prevToken(2) which should be "."
+  let index = 2;
+
+  // Walk backwards looking for metadata root
+  while (index <= currentTokenIndex) {
+    const token = tokens[currentTokenIndex - index];
+    if (!token) break;
+
+    if (token.text === kSampleMetadataVariable) {
+      return true;
+    } else if (token.text === "." || token.type === "variable") {
+      index++;
+    } else {
+      break; // Hit a non-metadata token
+    }
+  }
+
+  return false;
+};
+
+const makeMetadataKeyCompletion = (key: string): Completion => ({
+  label: key,
+  type: "property",
+  info: `Metadata property: ${key}`,
+  boost: 25,
+});
+
+const makeSampleIdCompletion = (id: string | number): Completion => ({
+  label: typeof id === "string" ? `"${id}"` : String(id),
+  type: "text",
+  info: `Sample ID: ${id}`,
+  boost: 25,
+});
+
+const makeMetadataValueCompletion = (value: any): Completion => {
+  let label: string;
+  if (typeof value === "string") {
+    label = `"${value}"`;
+  } else if (typeof value === "boolean") {
+    // Use filter expression constants for booleans
+    label = value ? "True" : "False";
+  } else if (value === null) {
+    label = "None";
+  } else {
+    label = String(value);
+  }
+
+  return {
+    label,
+    type: "text",
+    info: `Metadata value: ${value}`,
+    boost: 25,
+  };
+};
+
 /**
  * Generates completions for the filter expression. The main goal is to make the
  * sample filter intuitive for beginners and to provide a smooth experience for
@@ -137,6 +379,7 @@ const getMemberScoreItems = (
 export function getCompletions(
   context: CompletionContext,
   filterItems: SampleFilterItem[],
+  samples?: SampleSummary[],
 ): CompletionResult | null {
   const keywordCompletionItems = KEYWORDS.map(makeKeywordCompletion);
   const mathFunctionCompletionItems = MATH_FUNCTIONS.map(
@@ -145,7 +388,22 @@ export function getCompletions(
   const sampleFunctionCompletionItems = SAMPLE_FUNCTIONS.map(
     makeSampleFunctionCompletion,
   );
-  const sampleVariableCompletionItems = SAMPLE_VARIABLES.map(
+  // Filter sample variables based on available data
+  const availableSampleVariables = SAMPLE_VARIABLES.filter(([label]) => {
+    if (label === kSampleMetadataVariable) {
+      // Only include metadata if at least one sample has metadata
+      return (
+        samples &&
+        samples.some(
+          (sample) =>
+            sample.metadata && Object.keys(sample.metadata).length > 0,
+        )
+      );
+    }
+    return true;
+  });
+
+  const sampleVariableCompletionItems = availableSampleVariables.map(
     makeSampleVariableCompletion,
   );
   const variableCompletionItems = filterItems.map((item) =>
@@ -279,7 +537,7 @@ export function getCompletions(
       autoSpaceAfter: completingAtEnd,
     });
 
-  const descreteRelationCompletions = () =>
+  const discreteRelationCompletions = () =>
     makeCompletions(["==", "!=", "in", "not in"].map(makeKeywordCompletion), {
       enforceOrder: true,
       autoSpaceAfter: completingAtEnd,
@@ -305,9 +563,22 @@ export function getCompletions(
 
   // Member access
   if (prevToken(1)?.text === ".") {
-    const scorer = prevToken(2)?.text;
-    if (scorer) {
-      return memberAccessCompletions(getMemberScoreItems(filterItems, scorer));
+    const varName = prevToken(2)?.text;
+
+    // Check if this is metadata property access (metadata.* or metadata.*.*)
+    const metadataPath = buildMetadataPath(tokens, currentTokenIndex);
+    if (metadataPath !== null && samples) {
+      // Get completions for the current metadata path
+      const metadataKeys = Array.from(
+        getMetadataKeysForPath(samples, metadataPath),
+      );
+      const metadataCompletions = metadataKeys.map(makeMetadataKeyCompletion);
+      return makeCompletions(metadataCompletions, {
+        autocompleteInTheMiddle: true,
+        includeDefault: false,
+      });
+    } else if (varName) {
+      return memberAccessCompletions(getMemberScoreItems(filterItems, varName));
     }
   }
 
@@ -328,12 +599,31 @@ export function getCompletions(
 
   // Variable type-based relation suggestions
   if (prevToken(1)?.type === "variable") {
-    const scoreType = findFilterItem(1)?.scoreType || "";
+    const varName = prevToken(1)?.text;
 
+    // Check if this is a metadata property access (metadata.property or metadata.nested.property)
+    if (isMetadataProperty(tokens, currentTokenIndex)) {
+      // This is metadata.property - provide custom relation completions
+      return customRelationCompletions();
+    }
+
+    // Handle sample variables specially
+    if (varName === kSampleIdVariable) {
+      return discreteRelationCompletions();
+    }
+    if (varName === kSampleMetadataVariable) {
+      return customRelationCompletions();
+    }
+    if (varName === "has_error" || varName === "has_retries") {
+      return logicalOpCompletions();
+    }
+
+    // Handle score variables
+    const scoreType = findFilterItem(1)?.scoreType || "";
     switch (scoreType) {
       case kScoreTypePassFail:
       case kScoreTypeCategorical:
-        return descreteRelationCompletions();
+        return discreteRelationCompletions();
       case kScoreTypeNumeric:
         return continuousRelationCompletions();
       case kScoreTypeOther:
@@ -347,6 +637,68 @@ export function getCompletions(
 
   // RHS comparison suggestions
   if (prevToken(1)?.type === "relation") {
+    const varName = prevToken(2)?.text;
+
+    // Check if this is a metadata property comparison (relation after metadata.property or metadata.nested.property)
+    const metadataPropertyPath = buildMetadataPropertyPath(
+      tokens,
+      currentTokenIndex,
+    );
+    if (metadataPropertyPath !== null && samples) {
+      // This is metadata.property == ... - provide value completions for this property
+      const metadataValues = Array.from(
+        getMetadataPropertyValues(samples, metadataPropertyPath),
+      );
+
+      // Get the current query for prefix filtering
+      const currentQuery = currentToken?.text || "";
+
+      // Pre-filter values to only show prefix matches
+      const filteredValues = currentQuery
+        ? metadataValues.filter((value) => {
+            const label =
+              typeof value === "string"
+                ? `"${value}"`
+                : typeof value === "boolean"
+                  ? value
+                    ? "True"
+                    : "False"
+                  : value === null
+                    ? "None"
+                    : String(value);
+            return label.toLowerCase().startsWith(currentQuery.toLowerCase());
+          })
+        : metadataValues;
+
+      const metadataValueCompletions = filteredValues.map(
+        makeMetadataValueCompletion,
+      );
+      return makeCompletions(metadataValueCompletions, {
+        includeDefault: false,
+      });
+    }
+
+    // Sample ID completions
+    if (varName === kSampleIdVariable && samples) {
+      const sampleIds = Array.from(getSampleIds(samples));
+
+      // Get the current query for prefix filtering
+      const currentQuery = currentToken?.text || "";
+
+      // Pre-filter IDs to only show prefix matches
+      const filteredIds = currentQuery
+        ? sampleIds.filter((id) => {
+            const label = typeof id === "string" ? `"${id}"` : String(id);
+            return label.toLowerCase().startsWith(currentQuery.toLowerCase());
+          })
+        : sampleIds;
+
+      const sampleIdCompletions = filteredIds.map(makeSampleIdCompletion);
+      return makeCompletions(sampleIdCompletions, {
+        includeDefault: false,
+      });
+    }
+
     const item = findFilterItem(2);
     if (item?.categories?.length) {
       return rhsCompletions(item.categories);

inspect_ai/_view/www/src/app/samples/sample-tools/sample-filter/language.ts

@@ -1,3 +1,7 @@
+export const kSampleIdVariable = "id";
+export const kSampleMetadataVariable = "metadata";
+export const kSampleMetadataPrefix = kSampleMetadataVariable + ".";
+
 export const KEYWORDS: string[] = ["and", "or", "not", "in", "not in", "mod"];
 
 export const MATH_FUNCTIONS: [string, string][] = [
@@ -16,6 +20,8 @@ export const MATH_FUNCTIONS: [string, string][] = [
 export const SAMPLE_VARIABLES: [string, string][] = [
   ["has_error", "Checks if the sample has an error"],
   ["has_retries", "Checks if the sample has been retried"],
+  [kSampleIdVariable, "The unique identifier of the sample"],
+  [kSampleMetadataVariable, "Metadata associated with the sample"],
 ];
 
 export const SAMPLE_FUNCTIONS: [string, string][] = [

inspect_ai/_view/www/src/app/samples/sampleLimit.ts

@@ -1,9 +1,9 @@
-import { Type14 } from "../../@types/log";
+import { Type21 } from "../../@types/log";
 
 /**
  * Formats a limit message
  */
-export const sampleLimitMessage = (type: Type14): string => {
+export const sampleLimitMessage = (type: Type21): string => {
   switch (type) {
     case "operator":
       return "Sample terminated due to operator limit.";

inspect_ai/_view/www/src/app/samples/transcript/ModelEventView.tsx

@@ -216,7 +216,7 @@ const ToolsConfig: FC<ToolConfigProps> = ({ tools, toolChoice }) => {
       <div className={clsx(styles.toolConfig, "text-size-small")}>
         {toolEls}
       </div>
-      <div className={styles.toolChoice}>
+      <div className={clsx(styles.toolChoice, "text-size-small")}>
         <div className={clsx("text-style-label", "text-style-secondary")}>
           Tool Choice
         </div>

inspect_ai/_view/www/src/app/samples/transcript/SampleLimitEventView.tsx

@@ -1,6 +1,6 @@
 import clsx from "clsx";
 import { FC } from "react";
-import { SampleLimitEvent, Type10 } from "../../../@types/log";
+import { SampleLimitEvent, Type15 } from "../../../@types/log";
 import { ApplicationIcons } from "../../appearance/icons";
 import { EventPanel } from "./event/EventPanel";
 import { EventNode } from "./types";
@@ -17,12 +17,12 @@ export const SampleLimitEventView: FC<SampleLimitEventViewProps> = ({
   eventNode,
   className,
 }) => {
-  const resolve_title = (type: Type10) => {
+  const resolve_title = (type: Type15) => {
     switch (type) {
       case "custom":
         return "Custom Limit Exceeded";
       case "time":
-        return "Time Limit Execeeded";
+        return "Time Limit Exceeded";
       case "message":
         return "Message Limit Exceeded";
       case "token":
@@ -34,7 +34,7 @@ export const SampleLimitEventView: FC<SampleLimitEventViewProps> = ({
     }
   };
 
-  const resolve_icon = (type: Type10) => {
+  const resolve_icon = (type: Type15) => {
     switch (type) {
       case "custom":
         return ApplicationIcons.limits.custom;

inspect_ai/_view/www/src/app/samples/transcript/outline/OutlineRow.tsx

@@ -49,7 +49,7 @@ export const OutlineRow: FC<OutlineRowProps> = ({
       <div
         className={clsx(
           styles.eventRow,
-          "text-size-smallest",
+          "text-size-smaller",
           selected ? styles.selected : "",
         )}
         style={{ paddingLeft: `${node.depth * 0.4}em` }}

inspect_ai/_view/www/src/app/samples/transcript/outline/TranscriptOutline.tsx

@@ -16,7 +16,7 @@ import { useScrollTrack, useVirtuosoState } from "../../../../state/scrolling";
 import { useStore } from "../../../../state/store";
 import { flatTree } from "../transform/treeify";
 
-import { useSampleDetailNavigation } from "../../../routing/navigationHooks";
+import { useSampleDetailNavigation } from "../../../routing/sampleNavigation";
 import { kSandboxSignalName } from "../transform/fixups";
 import { OutlineRow } from "./OutlineRow";
 import styles from "./TranscriptOutline.module.css";

inspect_ai/_view/www/src/client/api/api-browser.ts

@@ -155,6 +155,29 @@ async function eval_log_sample_data(
   return result;
 }
 
+async function log_message(log_file: string, message: string) {
+  const params = new URLSearchParams();
+  params.append("log_file", log_file);
+  params.append("message", message);
+
+  const request: Request<void> = {
+    headers: {
+      "Content-Type": "text/plain",
+    },
+    parse: async (text: string) => {
+      if (text !== "") {
+        throw new Error(`Unexpected response from log_message: ${text}`);
+      }
+      return;
+    },
+  };
+  await apiRequest<void>(
+    "GET",
+    `/api/log-message?${params.toString()}`,
+    request,
+  );
+}
+
 interface Request<T> {
   headers?: Record<string, string>;
   body?: string;
@@ -288,7 +311,9 @@ const browserApi: LogViewAPI = {
   eval_log_size,
   eval_log_bytes,
   eval_log_headers,
+  log_message,
   download_file,
+
   open_log_file,
   eval_pending_samples,
   eval_log_sample_data,

inspect_ai/_view/www/src/client/api/api-http.ts

@@ -70,6 +70,9 @@ function simpleHttpAPI(logInfo: LogInfo): LogViewAPI {
 
       return undefined;
     },
+    log_message: async (log_file: string, message: string) => {
+      console.log(`[CLIENT MESSAGE] (${log_file}): ${message}`);
+    },
     eval_log: async (
       log_file: string,
       _headerOnly?: number,

inspect_ai/_view/www/src/client/api/api-vscode.ts

@@ -8,6 +8,7 @@ import {
   kMethodEvalLogHeaders,
   kMethodEvalLogs,
   kMethodEvalLogSize,
+  kMethodLogMessage,
   kMethodPendingSamples,
   kMethodSampleData,
   webViewJsonRpcClient,
@@ -147,6 +148,10 @@ async function eval_log_sample_data(
   }
 }
 
+async function log_message(log_file: string, message: string): Promise<void> {
+  await vscodeClient(kMethodLogMessage, [log_file, message]);
+}
+
 async function download_file() {
   throw Error("Downloading files is not supported in VS Code");
 }
@@ -167,6 +172,7 @@ const api: LogViewAPI = {
   eval_log_size,
   eval_log_bytes,
   eval_log_headers,
+  log_message,
   download_file,
   open_log_file,
   eval_pending_samples,

inspect_ai/_view/www/src/client/api/client-api.ts

@@ -335,6 +335,9 @@ export const clientApi = (api: LogViewAPI, log_file?: string): ClientAPI => {
     ) => {
       return api.download_file(download_file, file_contents);
     },
+    log_message: (log_file: string, message: string) => {
+      return api.log_message(log_file, message);
+    },
     get_log_pending_samples: api.eval_pending_samples
       ? get_log_pending_samples
       : undefined,

inspect_ai/_view/www/src/client/api/jsonrpc.ts

@@ -41,6 +41,7 @@ export const kMethodEvalLogBytes = "eval_log_bytes";
 export const kMethodEvalLogHeaders = "eval_log_headers";
 export const kMethodPendingSamples = "eval_log_pending_samples";
 export const kMethodSampleData = "eval_log_sample_data";
+export const kMethodLogMessage = "log_message";
 
 export const kJsonRpcParseError = -32700;
 export const kJsonRpcInvalidRequest = -32600;

inspect_ai/_view/www/src/client/api/types.ts

@@ -115,6 +115,7 @@ export interface SampleSummary {
   scores: Scores1;
   error?: string;
   limit?: string;
+  metadata?: Record<string, any>;
   completed?: boolean;
   retries?: number;
 }
@@ -149,6 +150,7 @@ export interface LogViewAPI {
     end: number,
   ) => Promise<Uint8Array>;
   eval_log_headers: (log_files: string[]) => Promise<EvalLog[]>;
+  log_message: (log_file: string, message: string) => Promise<void>;
   download_file: (
     filename: string,
     filecontents: string | Blob | ArrayBuffer | ArrayBufferView,
@@ -177,6 +179,7 @@ export interface ClientAPI {
     id: string | number,
     epoch: number,
   ) => Promise<EvalSample | undefined>;
+  log_message?: (log_file: string, message: string) => Promise<void>;
   download_file: (
     file_name: string,
     file_contents: string | Blob | ArrayBuffer | ArrayBufferView,