inspect-ai 0.3.103__py3-none-any.whl → 0.3.105__py3-none-any.whl

inspect_ai/_view/www/src/app/samples/chat/content-data/ContentDataView.tsx

@@ -0,0 +1,111 @@
+import { FC, ReactNode } from "react";
+import { WebSearch } from "./WebSearch";
+
+import clsx from "clsx";
+import { ContentData } from "../../../../@types/log";
+import { RecordTree } from "../../../content/RecordTree";
+import styles from "./ContentDataView.module.css";
+import { WebSearchContentData, WebSearchResults } from "./WebSearchResults";
+
+export interface ContentDataProps {
+  id: string;
+  contentData: ContentData;
+}
+
+interface RenderableData {
+  type: string;
+  name?: string;
+  [key: string]: any;
+}
+
+export const ContentDataView: FC<ContentDataProps> = ({ id, contentData }) => {
+  const renderableData = contentData.data as RenderableData;
+
+  const renderer = contentDataRenderers.find((r) =>
+    r.canRender(renderableData),
+  );
+
+  if (!renderer) {
+    const { encrypted_content, ...record } = renderableData;
+    return (
+      <div className={clsx(styles.contentData)}>
+        <RecordTree
+          id={`${id}-tree`}
+          record={record}
+          className={clsx(styles.data)}
+          defaultExpandLevel={0}
+        />
+      </div>
+    );
+  }
+
+  return (
+    <div className={clsx(styles.contentData)}>
+      {renderer.render(renderableData)}
+    </div>
+  );
+};
+
+// The following handles rendering of the content data based on its type
+// and name, allowing for different renderers to be used for different types of content data.
+
+interface ContentDataRenderer {
+  name: string;
+  canRender: (data: RenderableData) => boolean;
+  render: (data: RenderableData) => ReactNode;
+}
+
+const webSearchServerToolRenderer: ContentDataRenderer = {
+  name: "WebSearch",
+  canRender: (data: RenderableData) => {
+    return data.type === "server_tool_use" && data.name === "web_search";
+  },
+  render: (data: RenderableData): ReactNode => {
+    return <WebSearch query={data.input.query} />;
+  },
+};
+
+const webSearchResultsServerToolRenderer: ContentDataRenderer = {
+  name: "WebSearchResults",
+  canRender: (data: RenderableData) => {
+    return (
+      data.type === "web_search_tool_result" && Array.isArray(data.content)
+    );
+  },
+  render: (data: RenderableData): ReactNode => {
+    const results: WebSearchContentData[] =
+      data.content as WebSearchContentData[];
+    return <WebSearchResults results={results} />;
+  },
+};
+
+const serverToolRenderer: ContentDataRenderer = {
+  name: "ServerTool",
+  canRender: (data: RenderableData) => data.type === "server_tool_use",
+  render: (data: RenderableData): ReactNode => {
+    return (
+      <>
+        <div
+          className={clsx(
+            "text-style-label",
+            "text-style-secondary",
+            "text-size-smaller",
+          )}
+        >
+          Server Tool
+        </div>
+        <RecordTree
+          id={data.name || "server-tool"}
+          record={data}
+          className={clsx(styles.data)}
+        />
+      </>
+    );
+  },
+};
+
+export const contentDataRenderers: ContentDataRenderer[] = [
+  webSearchServerToolRenderer,
+  webSearchResultsServerToolRenderer,
+  serverToolRenderer,
+];

inspect_ai/_view/www/src/app/samples/chat/content-data/WebSearch.module.css

@@ -0,0 +1,10 @@
+.webSearch {
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  column-gap: 0.5em;
+  align-items: baseline;
+}
+
+.query {
+  font-family: var(--bs-font-monospace);
+}

inspect_ai/_view/www/src/app/samples/chat/content-data/WebSearch.tsx

@@ -0,0 +1,14 @@
+import clsx from "clsx";
+import { FC } from "react";
+import styles from "./WebSearch.module.css";
+
+export const WebSearch: FC<{ query: string }> = ({ query }) => {
+  return (
+    <div className={clsx(styles.webSearch, "text-size-smaller")}>
+      <span className={clsx("text-style-label", "text-style-secondary")}>
+        Web Search:
+      </span>
+      <span className={clsx(styles.query, "text-size-smallest")}>{query}</span>
+    </div>
+  );
+};

inspect_ai/_view/www/src/app/samples/chat/content-data/WebSearchResults.module.css

@@ -0,0 +1,19 @@
+.webSearch {
+  display: grid;
+  grid-template-columns: max-content 1fr;
+  column-gap: 0.5em;
+  align-items: baseline;
+}
+
+.query {
+  font-family: var(--bs-font-monospace);
+}
+
+.result a:hover {
+  text-decoration: underline;
+}
+
+.result a {
+  opacity: 0.8;
+  text-decoration: none;
+}

inspect_ai/_view/www/src/app/samples/chat/content-data/WebSearchResults.tsx

@@ -0,0 +1,49 @@
+import clsx from "clsx";
+import { FC } from "react";
+import styles from "./WebSearchResults.module.css";
+
+export interface WebSearchContentData {
+  title: string;
+  url: string;
+  page_age: string;
+}
+
+export const WebSearchResults: FC<{ results: WebSearchContentData[] }> = ({
+  results,
+}) => {
+  return (
+    <>
+      <div
+        className={clsx(
+          styles.label,
+          "text-style-label",
+          "text-style-secondary",
+          "text-size-smaller",
+        )}
+      >
+        Results
+      </div>
+
+      <ol className={clsx(styles.results, "text-size-smaller")}>
+        {results.map((result, index) => (
+          <li
+            key={index}
+            className={clsx(styles.result, "text-style-secondary")}
+          >
+            <a
+              href={result.url}
+              target="_blank"
+              rel="noopener noreferrer"
+              title={
+                result.url +
+                (result.page_age ? `\n(Age: ${result.page_age})` : "")
+              }
+            >
+              {result.title}
+            </a>
+          </li>
+        ))}
+      </ol>
+    </>
+  );
+};

inspect_ai/_view/www/src/app/samples/chat/messages.ts

@@ -4,6 +4,7 @@ import {
   ChatMessageTool,
   ChatMessageUser,
   ContentAudio,
+  ContentData,
   ContentImage,
   ContentReasoning,
   ContentText,
@@ -64,6 +65,7 @@ export const resolveMessages = (messages: Messages) => {
     | ContentAudio
     | ContentVideo
     | ContentReasoning
+    | ContentData
   )[] = [];
   for (const systemMessage of systemMessages) {
     const contents = Array.isArray(systemMessage.content)
@@ -78,6 +80,7 @@ export const resolveMessages = (messages: Messages) => {
     content: systemContent,
     source: "input",
     internal: null,
+    metadata: null,
   };
 
   // Converge them
@@ -117,19 +120,22 @@ const normalizeContent = (
     | ContentAudio
     | ContentVideo
     | ContentReasoning
+    | ContentData
     | string,
 ):
   | ContentText
   | ContentImage
   | ContentAudio
   | ContentVideo
-  | ContentReasoning => {
+  | ContentReasoning
+  | ContentData => {
   if (typeof content === "string") {
     return {
       type: "text",
       text: content,
       refusal: null,
       internal: null,
+      citations: null,
     };
   } else {
     return content;

inspect_ai/_view/www/src/app/samples/chat/tools/ToolCallView.tsx

@@ -2,6 +2,7 @@ import clsx from "clsx";
 import { FC, useMemo } from "react";
 import {
   ContentAudio,
+  ContentData,
   ContentImage,
   ContentReasoning,
   ContentText,
@@ -11,6 +12,7 @@ import {
 import { ContentTool } from "../../../../app/types";
 import ExpandablePanel from "../../../../components/ExpandablePanel";
 import { MessageContent } from "../MessageContent";
+import { defaultContext } from "../MessageContents";
 import styles from "./ToolCallView.module.css";
 import { ToolInput } from "./ToolInput";
 import { ToolTitle } from "./ToolTitle";
@@ -31,6 +33,7 @@ interface ToolCallViewProps {
     | ContentVideo
     | ContentTool
     | ContentReasoning
+    | ContentData
     | (
         | ContentText
         | ContentAudio
@@ -38,6 +41,7 @@ interface ToolCallViewProps {
         | ContentVideo
         | ContentTool
         | ContentReasoning
+        | ContentData
       )[];
   mode?: "compact";
 }
@@ -65,7 +69,8 @@ export const ToolCallView: FC<ToolCallViewProps> = ({
       | ContentImage
       | ContentVideo
       | ContentTool
-      | ContentReasoning,
+      | ContentReasoning
+      | ContentData,
   ) {
     if (value && typeof value === "object") {
       if (value.type === "image") {
@@ -105,6 +110,7 @@ export const ToolCallView: FC<ToolCallViewProps> = ({
   });
 
   const contents = mode !== "compact" ? input : input || functionCall;
+  const context = defaultContext();
   return (
     <div className={clsx(styles.toolCallView)}>
       <div>
@@ -127,7 +133,7 @@ export const ToolCallView: FC<ToolCallViewProps> = ({
           lines={15}
           className={clsx("text-size-small")}
         >
-          <MessageContent contents={normalizedContent} />
+          <MessageContent contents={normalizedContent} context={context} />
         </ExpandablePanel>
       ) : undefined}
     </div>
@@ -148,6 +154,7 @@ const normalizeContent = (
     | ContentVideo
     | ContentTool
     | ContentReasoning
+    | ContentData
     | (
         | ContentText
         | ContentImage
@@ -155,6 +162,7 @@ const normalizeContent = (
         | ContentVideo
         | ContentTool
         | ContentReasoning
+        | ContentData
       )[],
 ): (
   | ContentText
@@ -163,6 +171,7 @@ const normalizeContent = (
   | ContentVideo
   | ContentTool
   | ContentReasoning
+  | ContentData
 )[] => {
   if (Array.isArray(output)) {
     return output;
@@ -176,6 +185,7 @@ const normalizeContent = (
             text: String(output),
             refusal: null,
             internal: null,
+            citations: null,
           },
         ],
       },

inspect_ai/_view/www/src/app/samples/chat/types.ts

@@ -1 +1,5 @@
+import { Citations } from "../../../@types/log";
+
 export type ChatViewToolCallStyle = "compact" | "complete" | "omit";
+
+export type Citation = NonNullable<Citations>[number];

inspect_ai/_view/www/src/app/samples/list/SampleList.tsx

@@ -20,7 +20,7 @@ import clsx from "clsx";
 import { useProperty, useSampleDescriptor } from "../../../state/hooks";
 import { useVirtuosoState } from "../../../state/scrolling";
 import { useStore } from "../../../state/store";
-import { useSampleNavigation } from "../../routing/navigationHooks";
+import { useSampleNavigation } from "../../routing/sampleNavigation";
 import { SampleFooter } from "./SampleFooter";
 import { SampleHeader } from "./SampleHeader";
 import styles from "./SampleList.module.css";

inspect_ai/_view/www/src/app/samples/sample-tools/filters.ts

@@ -5,6 +5,7 @@ import { SampleSummary } from "../../../client/api/types";
 import { kScoreTypeBoolean } from "../../../constants";
 import { inputString } from "../../../utils/format";
 import { EvalDescriptor, ScoreDescriptor } from "../descriptor/types";
+import { kSampleMetadataPrefix } from "./sample-filter/language";
 
 export interface SampleFilterItem {
   shortName?: string;
@@ -100,10 +101,25 @@ const scoreVariables = (
   return variables;
 };
 
+const getNestedPropertyValue = (obj: any, path: string): any => {
+  const keys = path.split(".");
+  let current = obj;
+  for (const key of keys) {
+    if (current && typeof current === "object" && key in current) {
+      current = current[key];
+    } else {
+      return undefined;
+    }
+  }
+  return current;
+};
+
 const sampleVariables = (sample: SampleSummary): Record<string, unknown> => {
   return {
     has_error: !!sample.error,
     has_retries: sample.retries !== undefined && sample.retries > 0,
+    id: sample.id,
+    metadata: sample.metadata,
   };
 };
 
@@ -217,6 +233,12 @@ export const filterExpression = (
         const value = get(name);
         return value;
       }
+      // Handle metadata property access
+      if (name.startsWith(kSampleMetadataPrefix)) {
+        const propertyPath = name.substring(kSampleMetadataPrefix.length);
+        const metadata = sample.metadata || {};
+        return getNestedPropertyValue(metadata, propertyPath);
+      }
       // Score variables exist only if the sample completed successfully.
       return sample.error ? undefined : get(name);
     };
@@ -240,6 +262,10 @@ export const filterExpression = (
       const errorObj = error as any as Record<string, unknown>;
       const propertyName: string = (errorObj["propertyName"] as string) || "";
       if (propertyName) {
+        // Don't show errors for metadata properties - they might not exist in all samples
+        if (propertyName.startsWith(kSampleMetadataPrefix)) {
+          return { matches: false, error: undefined };
+        }
         const regex = new RegExp(`\\b${propertyName}\\b`);
         const match = regex.exec(filterValue);
         if (match) {

inspect_ai/_view/www/src/app/samples/sample-tools/sample-filter/SampleFilter.tsx

@@ -34,6 +34,8 @@ Filter samples by:
   • Samples with errors: has_error
   • Input, target and error regex search: input_contains, target_contains, error_contains
   • Samples that have been retried: has_retries
+  • Sample Id: e.g. "id == 'sample123'"
+  • Sample metadata: e.g. "metadata.key == 'value'"
 
 Supported expressions:
   • Arithmetic: +, -, *, /, mod, ^
@@ -93,6 +95,12 @@ const editorTheme = EditorView.theme({
   ".cm-scroller": {
     overflow: "hidden",
   },
+  ".cm-line": {
+    "font-size": "var(--inspect-font-size-smallest) !important",
+  },
+  ".token": {
+    "font-size": "var(--inspect-font-size-smallest) !important",
+  },
 });
 
 const ensureOneLine = (tr: Transaction): TransactionSpec => {
@@ -145,6 +153,9 @@ export const SampleFilter: FC<SampleFilterProps> = () => {
 
   const filter = useStore((state) => state.log.filter);
   const filterError = useStore((state) => state.log.filterError);
+  const samples = useStore(
+    (state) => state.log.selectedLogSummary?.sampleSummaries,
+  );
   const setFilter = useStore((state) => state.logActions.setFilter);
 
   const handleFocus = useCallback((event: FocusEvent, view: EditorView) => {
@@ -156,10 +167,10 @@ export const SampleFilter: FC<SampleFilterProps> = () => {
   const makeAutocompletion = useCallback(
     () =>
       autocompletion({
-        override: [(context) => getCompletions(context, filterItems)],
+        override: [(context) => getCompletions(context, filterItems, samples)],
         activateOnCompletion: (c) => c.label.endsWith(" "),
       }),
-    [],
+    [filterItems, samples],
   );
 
   const makeLinter = useCallback(
@@ -240,7 +251,7 @@ export const SampleFilter: FC<SampleFilterProps> = () => {
       effects:
         autocompletionCompartment.current.reconfigure(makeAutocompletion()),
     });
-  }, [filterItems]);
+  }, [filterItems, samples]);
 
   useEffect(() => {
     editorViewRef.current?.dispatch({