inspect-ai 0.3.51__py3-none-any.whl → 0.3.53__py3-none-any.whl

inspect_ai/_view/www/App.css

@@ -711,6 +711,19 @@ pre[class*="language-"].tool-output,
   background-color: #333333;
 }
 
+pre[class*="language-"].tool-output {
+  border: none !important;
+  box-shadow: none !important;
+  border-radius: var(--bs-border-radius) !important;
+}
+
+.vscode-dark pre.jsonPanel {
+  background: none !important;
+  border: none !important;
+  box-shadow: none !important;
+  border-radius: var(--bs-border-radius) !important;  
+}
+
 
 /* jsondiffpatch */
 

inspect_ai/_view/www/dist/assets/index.css

@@ -14984,6 +14984,19 @@ pre[class*="language-"].tool-output,
   background-color: #333333;
 }
 
+pre[class*="language-"].tool-output {
+  border: none !important;
+  box-shadow: none !important;
+  border-radius: var(--bs-border-radius) !important;
+}
+
+.vscode-dark pre.jsonPanel {
+  background: none !important;
+  border: none !important;
+  box-shadow: none !important;
+  border-radius: var(--bs-border-radius) !important;  
+}
+
 
 /* jsondiffpatch */
 

inspect_ai/_view/www/dist/assets/index.js

@@ -15251,21 +15251,10 @@ const ToolCallView = ({
   output,
   mode
 }) => {
-  const icon = mode === "compact" ? "" : m$1`<i
-          class="bi bi-tools"
-          style=${{
-    marginRight: "0.2rem",
-    opacity: "0.4"
-  }}
-        ></i>`;
-  const codeIndent = mode === "compact" ? "" : "";
   return m$1`<div>
-    ${icon}
-    ${!view || view.title ? m$1`<code style=${{ fontSize: FontSize.small }}
-          >${(view == null ? void 0 : view.title) || functionCall}</code
-        >` : ""}
+    ${mode !== "compact" && (!view || view.title) ? m$1`<${ToolTitle} title=${(view == null ? void 0 : view.title) || functionCall} />` : ""}
     <div>
-      <div style=${{ marginLeft: `${codeIndent}` }}>
+      <div>
         <${ToolInput}
           type=${inputType}
           contents=${input}
@@ -15274,12 +15263,39 @@ const ToolCallView = ({
         />
         ${output ? m$1`
               <${ExpandablePanel} collapse=${true} border=${true} lines=${15}>
-              <${MessageContent} contents=${output} />
+              <${MessageContent} contents=${normalizeContent$1(output)} />
               </${ExpandablePanel}>` : ""}
       </div>
     </div>
   </div>`;
 };
+const ToolTitle = ({ title }) => {
+  return m$1` <i
+      class="bi bi-tools"
+      style=${{
+    marginRight: "0.2rem",
+    opacity: "0.4"
+  }}
+    ></i>
+    <code style=${{ fontSize: FontSize.small }}>${title}</code>`;
+};
+const normalizeContent$1 = (output) => {
+  if (Array.isArray(output)) {
+    return output;
+  } else {
+    return [
+      {
+        type: "tool",
+        content: [
+          {
+            type: "text",
+            text: String(output)
+          }
+        ]
+      }
+    ];
+  }
+};
 const ToolInput = ({ type, contents, view, style }) => {
   if (!contents && !(view == null ? void 0 : view.content)) {
     return "";
@@ -15455,8 +15471,7 @@ const extractInput = (inputKey, args) => {
     args: []
   };
 };
-const MessageContent = (props) => {
-  const { contents } = props;
+const MessageContent = ({ contents }) => {
   if (Array.isArray(contents)) {
     return contents.map((content, index) => {
       if (typeof content === "string") {
@@ -19422,21 +19437,15 @@ const LoggerEventView = ({ id, event, style }) => {
 };
 const kPrismRenderMaxSize = 25e4;
 const JSONPanel = ({ id, json, data, simple, style }) => {
-  const sourceCode = json || JSON.stringify(data, void 0, 2);
   const codeRef = A();
-  if (codeRef.current) {
+  const sourceCode = T(() => {
+    return json || JSON.stringify(data, void 0, 2);
+  }, [json, data]);
+  y(() => {
     if (sourceCode.length < kPrismRenderMaxSize) {
-      codeRef.current.innerHTML = Prism$1.highlight(
-        sourceCode,
-        Prism$1.languages.javascript,
-        "javacript"
-      );
-    } else {
-      const textNode = document.createTextNode(sourceCode);
-      codeRef.current.innerText = "";
-      codeRef.current.appendChild(textNode);
+      Prism$1.highlightElement(codeRef.current);
     }
-  }
+  }, [sourceCode]);
   return m$1`<div>
     <pre
       style=${{
@@ -19446,16 +19455,18 @@ const JSONPanel = ({ id, json, data, simple, style }) => {
     borderRadius: simple ? void 0 : "var(--bs-border-radius)",
     ...style
   }}
+      class="jsonPanel"
     >
     <code 
       id=${id}
       ref=${codeRef}
-      class="sourceCode-json" 
+      class="sourceCode language-javascript" 
       style=${{
     fontSize: FontSize.small,
     whiteSpace: "pre-wrap",
     wordWrap: "anywhere"
   }}>
+      ${sourceCode}
     </code>
     </pre>
   </div>`;
@@ -19569,6 +19580,7 @@ const decisionIcon = (decision) => {
   }
 };
 const ToolEventView = ({ id, event, style, depth }) => {
+  var _a2;
   const { input, functionCall, inputType } = resolveToolInput(
     event.function,
     event.arguments
@@ -19576,10 +19588,10 @@ const ToolEventView = ({ id, event, style, depth }) => {
   const approvalEvent = event.events.find((e2) => {
     return e2.event === "approval";
   });
-  const title = `Tool: ${event.function}`;
+  const title = `Tool: ${((_a2 = event.view) == null ? void 0 : _a2.title) || event.function}`;
   return m$1`
   <${EventPanel} id=${id} title="${title}" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.solvers.use_tools} style=${style}>  
-  <div name="Summary" style=${{ margin: "0.5em 0" }}>
+  <div name="Summary" style=${{ margin: "0.5em 0", width: "100%" }}>
     <${ToolCallView}
       functionCall=${functionCall}
       input=${input}
@@ -20225,7 +20237,7 @@ const metadataViewsForSample = (id, sample) => {
   }
   return sampleMetadatas;
 };
-const SampleSummary = ({ id, sample, style, sampleDescriptor }) => {
+const SampleSummary = ({ parent_id, sample, style, sampleDescriptor }) => {
   const input = (sampleDescriptor == null ? void 0 : sampleDescriptor.messageShape.normalized.input) > 0 ? Math.max(0.15, sampleDescriptor.messageShape.normalized.input) : 0;
   const target = (sampleDescriptor == null ? void 0 : sampleDescriptor.messageShape.normalized.target) > 0 ? Math.max(0.15, sampleDescriptor.messageShape.normalized.target) : 0;
   const answer = (sampleDescriptor == null ? void 0 : sampleDescriptor.messageShape.normalized.answer) > 0 ? Math.max(0.15, sampleDescriptor.messageShape.normalized.answer) : 0;
@@ -20246,7 +20258,7 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => {
   const columns = [];
   columns.push({
     label: "Id",
-    value: id,
+    value: sample.id,
     size: `${idSize}em`
   });
   columns.push({
@@ -20267,7 +20279,10 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => {
       clamp: true
     });
   }
-  const fullAnswer = sample && sampleDescriptor ? sampleDescriptor.selectedScorer(sample).answer() : void 0;
+  const fullAnswer = sample && sampleDescriptor ? (
+    // @ts-ignore
+    sampleDescriptor.selectedScorer(sample).answer()
+  ) : void 0;
   if (fullAnswer) {
     columns.push({
       label: "Answer",
@@ -20293,13 +20308,17 @@ const SampleSummary = ({ id, sample, style, sampleDescriptor }) => {
     value: sample.error ? m$1`<${FlatSampleError}
           message=${sample.error.message}
           style=${{ marginTop: "0.4rem" }}
-        />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render(),
+        />` : (
+      // TODO: Cleanup once the PR lands which makes sample / sample summary share common interface
+      // @ts-ignore
+      sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render()
+    ),
     size: "minmax(2em, auto)",
     center: true
   });
   return m$1`
     <div
-      id=${`sample-heading-${id}`}
+      id=${`sample-heading-${parent_id}`}
       style=${{
     display: "grid",
     gridTemplateColumns: `${columns.map((col) => {
@@ -20601,7 +20620,7 @@ const SampleList = (props) => {
     [selectedIndex]
   );
   const listStyle = { ...style, flex: "1", overflowY: "auto", outline: "none" };
-  const { limit, answer } = gridColumns(sampleDescriptor);
+  const { limit, answer, target } = gridColumns(sampleDescriptor);
   const headerRow = m$1`<div
     style=${{
     display: "grid",
@@ -20616,7 +20635,7 @@ const SampleList = (props) => {
   >
     <div>Id</div>
     <div>Input</div>
-    <div>Target</div>
+    <div>${target !== "0" ? "Target" : ""}</div>
     <div>${answer !== "0" ? "Answer" : ""}</div>
     <div>${limit !== "0" ? "Limit" : ""}</div>
     <div style=${{ justifySelf: "center" }}>Score</div>
@@ -24620,7 +24639,9 @@ const ResultsPanel = ({ results }) => {
       flexWrap: "wrap",
       justifyContent: "end",
       height: "100%",
-      alignItems: "center"
+      alignItems: "center",
+      maxHeight: "15em",
+      overflow: "scroll"
     }}
     >
       ${metrics.map((metric, i) => {
@@ -24638,7 +24659,9 @@ const ResultsPanel = ({ results }) => {
       alignItems: "center",
       marginTop: "0.2rem",
       paddingBottom: "0.4rem",
-      rowGap: "1em"
+      rowGap: "1em",
+      maxHeight: "15em",
+      overflow: "scroll"
     }}
     >
       ${(_b2 = results == null ? void 0 : results.scores) == null ? void 0 : _b2.map((score, index) => {
@@ -25449,22 +25472,35 @@ const createsSamplesDescriptor = (scorers, samples, epochs, selectedScore) => {
         if (!sample || !sample.scores) {
           return [];
         }
-        scorers.map((score2) => {
+        const scoreNames = scorers.map((score2) => {
           return score2.name;
         });
         const sampleScorer = sample.scores[scorer];
         const scoreVal = sampleScorer.value;
         if (typeof scoreVal === "object") {
           const names = Object.keys(scoreVal);
-          const scores = names.map((name) => {
-            return {
-              name,
-              rendered: () => {
-                return scoreDescriptor.render(scoreVal[name]);
+          if (names.find((name) => {
+            return scoreNames.includes(name);
+          })) {
+            const scores = names.map((name) => {
+              return {
+                name,
+                rendered: () => {
+                  return scoreDescriptor.render(scoreVal[name]);
+                }
+              };
+            });
+            return scores;
+          } else {
+            return [
+              {
+                name: scorer,
+                rendered: () => {
+                  return scoreDescriptor.render(scoreVal);
+                }
               }
-            };
-          });
-          return scores;
+            ];
+          }
         } else {
           return [
             {
@@ -26210,8 +26246,12 @@ function App({
           }
         }
       } catch (e2) {
-        console.log(e2);
-        setStatus({ loading: false, error: e2 });
+        if (e2.message === "Load failed" || e2.message === "Failed to fetch") {
+          setStatus({ loading: false });
+        } else {
+          console.log(e2);
+          setStatus({ loading: false, error: e2 });
+        }
       }
       setHeadersLoading(false);
     };
@@ -26439,8 +26479,11 @@ function App({
       }
       new ClipboardJS(".clipboard-button,.copy-button");
       if (pollForLogs) {
-        setInterval(() => {
-          api2.client_events().then(async (events) => {
+        let retryDelay = 1e3;
+        const maxRetryDelay = 6e4;
+        const pollEvents = async () => {
+          try {
+            const events = await api2.client_events();
             if (events.includes("reload")) {
               window.location.reload();
             }
@@ -26449,8 +26492,15 @@ function App({
               setLogs(logs2);
               setSelectedLogIndex(0);
             }
-          });
-        }, 1e3);
+            retryDelay = 1e3;
+          } catch (error2) {
+            console.error("Error fetching client events:", error2);
+            retryDelay = Math.min(retryDelay * 2, maxRetryDelay);
+          } finally {
+            setTimeout(pollEvents, retryDelay);
+          }
+        };
+        pollEvents();
       }
     };
     loadLogsAndState();

inspect_ai/_view/www/src/App.mjs

@@ -480,8 +480,13 @@ export function App({
           }
         }
       } catch (e) {
-        console.log(e);
-        setStatus({ loading: false, error: e });
+        if (e.message === "Load failed" || e.message === "Failed to fetch") {
+          // This will happen if the server disappears (e.g. inspect view is terminated)
+          setStatus({ loading: false });
+        } else {
+          console.log(e);
+          setStatus({ loading: false, error: e });
+        }
       }
 
       setHeadersLoading(false);
@@ -774,18 +779,38 @@ export function App({
       new ClipboardJS(".clipboard-button,.copy-button");
 
       if (pollForLogs) {
-        setInterval(() => {
-          api.client_events().then(async (events) => {
+        let retryDelay = 1000;
+        const maxRetryDelay = 60000;
+
+        const pollEvents = async () => {
+          try {
+            const events = await api.client_events();
+
             if (events.includes("reload")) {
               window.location.reload();
             }
+
             if (events.includes("refresh-evals")) {
               const logs = await load();
               setLogs(logs);
               setSelectedLogIndex(0);
             }
-          });
-        }, 1000);
+
+            // Reset delay after a successful call
+            retryDelay = 1000;
+          } catch (error) {
+            console.error("Error fetching client events:", error);
+
+            // Exponential backoff with capping
+            retryDelay = Math.min(retryDelay * 2, maxRetryDelay);
+          } finally {
+            // Schedule the next poll
+            setTimeout(pollEvents, retryDelay);
+          }
+        };
+
+        // Start polling
+        pollEvents();
       }
     };
 

inspect_ai/_view/www/src/Types.mjs

@@ -32,3 +32,9 @@
 /**
  * @typedef {"none" | "single" | "many"} SampleMode
  */
+
+/**
+ * @typedef {Object} ContentTool
+ * @property {"tool"} type
+ * @property {(import("./types/log").ContentImage | import("./types/log").ContentText)[]} content
+ */

inspect_ai/_view/www/src/components/JsonPanel.mjs

@@ -4,7 +4,7 @@ import Prism from "prismjs";
 import "prismjs/components/prism-json";
 
 import { html } from "htm/preact";
-import { useRef } from "preact/hooks";
+import { useEffect, useMemo, useRef } from "preact/hooks";
 import { FontSize } from "../appearance/Fonts.mjs";
 
 const kPrismRenderMaxSize = 250000;
@@ -22,25 +22,17 @@ const kPrismRenderMaxSize = 250000;
  * @returns {import('preact').JSX.Element} The rendered component.
  */
 export const JSONPanel = ({ id, json, data, simple, style }) => {
-  const sourceCode = json || JSON.stringify(data, undefined, 2);
   const codeRef = useRef();
 
-  if (codeRef.current) {
+  const sourceCode = useMemo(() => {
+    return json || JSON.stringify(data, undefined, 2);
+  }, [json, data]);
+
+  useEffect(() => {
     if (sourceCode.length < kPrismRenderMaxSize) {
-      // @ts-ignore
-      codeRef.current.innerHTML = Prism.highlight(
-        sourceCode,
-        Prism.languages.javascript,
-        "javacript",
-      );
-    } else {
-      const textNode = document.createTextNode(sourceCode);
-      // @ts-ignore
-      codeRef.current.innerText = "";
-      // @ts-ignore
-      codeRef.current.appendChild(textNode);
+      Prism.highlightElement(codeRef.current);
     }
-  }
+  }, [sourceCode]);
 
   return html`<div>
     <pre
@@ -51,16 +43,18 @@ export const JSONPanel = ({ id, json, data, simple, style }) => {
         borderRadius: simple ? undefined : "var(--bs-border-radius)",
         ...style,
       }}
+      class="jsonPanel"
     >
     <code 
       id=${id}
       ref=${codeRef}
-      class="sourceCode-json" 
+      class="sourceCode language-javascript" 
       style=${{
       fontSize: FontSize.small,
       whiteSpace: "pre-wrap",
       wordWrap: "anywhere",
     }}>
+      ${sourceCode}
     </code>
     </pre>
   </div>`;

inspect_ai/_view/www/src/components/MessageContent.mjs

@@ -2,8 +2,15 @@ import { html } from "htm/preact";
 import { MarkdownDiv } from "./MarkdownDiv.mjs";
 import { ToolOutput } from "./Tools.mjs";
 
-export const MessageContent = (props) => {
-  const { contents } = props;
+/**
+ * Renders message content based on its type.
+ * Supports rendering strings, images, and tools using specific renderers.
+ *
+ * @param {Object} props - The props object.
+ * @param {string|string[]| (import("../types/log").ContentText | import("../types/log").ContentImage | import("../Types.mjs").ContentTool)[]} props.contents - The content or array of contents to render.
+ * @returns {import("preact").JSX.Element | import("preact").JSX.Element[]} The component.
+ */
+export const MessageContent = ({ contents }) => {
   if (Array.isArray(contents)) {
     return contents.map((content, index) => {
       if (typeof content === "string") {

inspect_ai/_view/www/src/components/Tools.mjs

@@ -63,26 +63,12 @@ export const ToolCallView = ({
   output,
   mode,
 }) => {
-  const icon =
-    mode === "compact"
-      ? ""
-      : html`<i
-          class="bi bi-tools"
-          style=${{
-            marginRight: "0.2rem",
-            opacity: "0.4",
-          }}
-        ></i>`;
-  const codeIndent = mode === "compact" ? "" : "";
   return html`<div>
-    ${icon}
-    ${!view || view.title
-      ? html`<code style=${{ fontSize: FontSize.small }}
-          >${view?.title || functionCall}</code
-        >`
+    ${mode !== "compact" && (!view || view.title)
+      ? html`<${ToolTitle} title=${view?.title || functionCall} />`
       : ""}
     <div>
-      <div style=${{ marginLeft: `${codeIndent}` }}>
+      <div>
         <${ToolInput}
           type=${inputType}
           contents=${input}
@@ -92,7 +78,7 @@ export const ToolCallView = ({
         ${output
           ? html`
               <${ExpandablePanel} collapse=${true} border=${true} lines=${15}>
-              <${MessageContent} contents=${output} />
+              <${MessageContent} contents=${normalizeContent(output)} />
               </${ExpandablePanel}>`
           : ""}
       </div>
@@ -100,6 +86,48 @@ export const ToolCallView = ({
   </div>`;
 };
 
+/**
+ * Renders the ToolCallView component.
+ *
+ * @param {Object} props - The parameters for the component.
+ * @param {string} props.title - The title for the tool call
+ * @returns {import("preact").JSX.Element} The SampleTranscript component.
+ */
+const ToolTitle = ({ title }) => {
+  return html` <i
+      class="bi bi-tools"
+      style=${{
+        marginRight: "0.2rem",
+        opacity: "0.4",
+      }}
+    ></i>
+    <code style=${{ fontSize: FontSize.small }}>${title}</code>`;
+};
+
+/**
+ * Renders the ToolCallView component.
+ *
+ * @param {string | number | boolean | (import("../types/log").ContentText | import("../types/log").ContentImage)[]} output - The tool output
+ * @returns {(import("../Types.mjs").ContentTool | import("../types/log").ContentText | import("../types/log").ContentImage)[]} The SampleTranscript component.
+ */
+const normalizeContent = (output) => {
+  if (Array.isArray(output)) {
+    return output;
+  } else {
+    return [
+      {
+        type: "tool",
+        content: [
+          {
+            type: "text",
+            text: String(output),
+          },
+        ],
+      },
+    ];
+  }
+};
+
 /**
  * Renders the ToolInput component.
  *

inspect_ai/_view/www/src/navbar/Navbar.mjs

@@ -255,6 +255,8 @@ const ResultsPanel = ({ results }) => {
         justifyContent: "end",
         height: "100%",
         alignItems: "center",
+        maxHeight: "15em",
+        overflow: "scroll",
       }}
     >
       ${metrics.map((metric, i) => {
@@ -273,6 +275,8 @@ const ResultsPanel = ({ results }) => {
         marginTop: "0.2rem",
         paddingBottom: "0.4rem",
         rowGap: "1em",
+        maxHeight: "15em",
+        overflow: "scroll",
       }}
     >
       ${results?.scores?.map((score, index) => {
@@ -285,6 +289,14 @@ const ResultsPanel = ({ results }) => {
   }
 };
 
+/** Renders a Vertial Metric
+ *
+ * @param {Object} props - The parameters for the component.
+ * @param {import("../types/log").EvalMetric} props.metric - The metric
+ * @param {boolean} props.isFirst - Whether this is the first metric
+ *
+ * @returns {import("preact").JSX.Element} The TranscriptView component.
+ */
 const VerticalMetric = ({ metric, isFirst }) => {
   const reducer_component = metric.reducer
     ? html` <div