inspect-ai 0.3.63__py3-none-any.whl → 0.3.65__py3-none-any.whl

inspect_ai/_view/www/log-schema.json

@@ -157,6 +157,7 @@
       "type": "object"
     },
     "ChatCompletionChoice": {
+      "description": "Choice generated for completion.",
       "properties": {
         "message": {
           "$ref": "#/$defs/ChatMessageAssistant"
@@ -196,7 +197,14 @@
       "additionalProperties": false
     },
     "ChatMessageAssistant": {
+      "description": "Assistant chat message.",
       "properties": {
+        "role": {
+          "const": "assistant",
+          "default": "assistant",
+          "title": "Role",
+          "type": "string"
+        },
         "content": {
           "anyOf": [
             {
@@ -240,12 +248,6 @@
           "default": null,
           "title": "Source"
         },
-        "role": {
-          "const": "assistant",
-          "default": "assistant",
-          "title": "Role",
-          "type": "string"
-        },
         "tool_calls": {
           "anyOf": [
             {
@@ -275,9 +277,9 @@
         }
       },
       "required": [
+        "role",
         "content",
         "source",
-        "role",
         "tool_calls",
         "reasoning"
       ],
@@ -286,7 +288,14 @@
       "additionalProperties": false
     },
     "ChatMessageSystem": {
+      "description": "System chat message.",
       "properties": {
+        "role": {
+          "const": "system",
+          "default": "system",
+          "title": "Role",
+          "type": "string"
+        },
         "content": {
           "anyOf": [
             {
@@ -329,25 +338,26 @@
           ],
           "default": null,
           "title": "Source"
-        },
-        "role": {
-          "const": "system",
-          "default": "system",
-          "title": "Role",
-          "type": "string"
         }
       },
       "required": [
+        "role",
         "content",
-        "source",
-        "role"
+        "source"
       ],
       "title": "ChatMessageSystem",
       "type": "object",
       "additionalProperties": false
     },
     "ChatMessageTool": {
+      "description": "Tool chat message.",
       "properties": {
+        "role": {
+          "const": "tool",
+          "default": "tool",
+          "title": "Role",
+          "type": "string"
+        },
         "content": {
           "anyOf": [
             {
@@ -391,12 +401,6 @@
           "default": null,
           "title": "Source"
         },
-        "role": {
-          "const": "tool",
-          "default": "tool",
-          "title": "Role",
-          "type": "string"
-        },
         "tool_call_id": {
           "anyOf": [
             {
@@ -434,9 +438,9 @@
         }
       },
       "required": [
+        "role",
         "content",
         "source",
-        "role",
         "tool_call_id",
         "function",
         "error"
@@ -446,7 +450,14 @@
       "additionalProperties": false
     },
     "ChatMessageUser": {
+      "description": "User chat message.",
       "properties": {
+        "role": {
+          "const": "user",
+          "default": "user",
+          "title": "Role",
+          "type": "string"
+        },
         "content": {
           "anyOf": [
             {
@@ -490,12 +501,6 @@
           "default": null,
           "title": "Source"
         },
-        "role": {
-          "const": "user",
-          "default": "user",
-          "title": "Role",
-          "type": "string"
-        },
         "tool_call_id": {
           "anyOf": [
             {
@@ -513,9 +518,9 @@
         }
       },
       "required": [
+        "role",
         "content",
         "source",
-        "role",
         "tool_call_id"
       ],
       "title": "ChatMessageUser",
@@ -523,6 +528,7 @@
       "additionalProperties": false
     },
     "ContentAudio": {
+      "description": "Audio content.",
       "properties": {
         "type": {
           "const": "audio",
@@ -553,6 +559,7 @@
       "additionalProperties": false
     },
     "ContentImage": {
+      "description": "Image content.",
       "properties": {
         "type": {
           "const": "image",
@@ -585,6 +592,7 @@
       "additionalProperties": false
     },
     "ContentText": {
+      "description": "Text content.",
       "properties": {
         "type": {
           "const": "text",
@@ -606,6 +614,7 @@
       "additionalProperties": false
     },
     "ContentVideo": {
+      "description": "Video content.",
       "properties": {
         "type": {
           "const": "video",
@@ -677,6 +686,7 @@
       "additionalProperties": false
     },
     "EvalConfig": {
+      "description": "Configuration used for evaluation.",
       "properties": {
         "limit": {
           "anyOf": [
@@ -954,6 +964,7 @@
       "additionalProperties": false
     },
     "EvalDataset": {
+      "description": "Dataset used for evaluation.",
       "properties": {
         "name": {
           "anyOf": [
@@ -1038,6 +1049,7 @@
       "additionalProperties": false
     },
     "EvalError": {
+      "description": "Eval error details.",
       "properties": {
         "message": {
           "title": "Message",
@@ -1062,6 +1074,7 @@
       "additionalProperties": false
     },
     "EvalMetric": {
+      "description": "Metric for evaluation score.",
       "properties": {
         "name": {
           "title": "Name",
@@ -1078,8 +1091,8 @@
           ],
           "title": "Value"
         },
-        "options": {
-          "title": "Options",
+        "params": {
+          "title": "Params",
           "type": "object"
         },
         "metadata": {
@@ -1098,7 +1111,7 @@
       "required": [
         "name",
         "value",
-        "options",
+        "params",
         "metadata"
       ],
       "title": "EvalMetric",
@@ -1106,6 +1119,7 @@
       "additionalProperties": false
     },
     "EvalPlan": {
+      "description": "Plan (solvers) used in evaluation.",
       "properties": {
         "name": {
           "default": "plan",
@@ -1171,6 +1185,7 @@
       "additionalProperties": false
     },
     "EvalPlanStep": {
+      "description": "Solver step.",
       "properties": {
         "solver": {
           "title": "Solver",
@@ -1190,6 +1205,7 @@
       "additionalProperties": false
     },
     "EvalResults": {
+      "description": "Scoring results from evaluation.",
       "properties": {
         "total_samples": {
           "default": 0,
@@ -1233,6 +1249,7 @@
       "additionalProperties": false
     },
     "EvalRevision": {
+      "description": "Git revision for evaluation.",
       "properties": {
         "type": {
           "const": "git",
@@ -1258,6 +1275,7 @@
       "additionalProperties": false
     },
     "EvalSample": {
+      "description": "Sample from evaluation task.",
       "properties": {
         "id": {
           "anyOf": [
@@ -1526,6 +1544,7 @@
       "additionalProperties": false
     },
     "EvalSampleLimit": {
+      "description": "Limit encontered by sample.",
       "properties": {
         "type": {
           "enum": [
@@ -1553,6 +1572,7 @@
       "additionalProperties": false
     },
     "EvalSampleReductions": {
+      "description": "Score reductions.",
       "properties": {
         "scorer": {
           "title": "Scorer",
@@ -1588,6 +1608,7 @@
       "additionalProperties": false
     },
     "EvalSampleScore": {
+      "description": "Score and sample_id scored.",
       "properties": {
         "value": {
           "anyOf": [
@@ -1711,6 +1732,7 @@
       "additionalProperties": false
     },
     "EvalScore": {
+      "description": "Score for evaluation task.",
       "properties": {
         "name": {
           "title": "Name",
@@ -1769,6 +1791,7 @@
       "additionalProperties": false
     },
     "EvalSpec": {
+      "description": "Eval target and configuration.",
       "properties": {
         "run_id": {
           "title": "Run Id",
@@ -1945,6 +1968,7 @@
       "additionalProperties": false
     },
     "EvalStats": {
+      "description": "Timing and usage statistics.",
       "properties": {
         "started_at": {
           "title": "Started At",
@@ -1972,7 +1996,7 @@
       "additionalProperties": false
     },
     "GenerateConfig": {
-      "description": "Base class for model generation configs.",
+      "description": "Model generation options.",
       "properties": {
         "max_retries": {
           "anyOf": [
@@ -2321,6 +2345,18 @@
           "title": "Event",
           "type": "string"
         },
+        "source": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Source"
+        },
         "data": {
           "$ref": "#/$defs/JsonValue"
         }
@@ -2329,6 +2365,7 @@
         "timestamp",
         "pending",
         "event",
+        "source",
         "data"
       ],
       "title": "InfoEvent",
@@ -2474,6 +2511,7 @@
       "additionalProperties": false
     },
     "LoggingMessage": {
+      "description": "Message written to Python log.",
       "properties": {
         "name": {
           "anyOf": [
@@ -2772,6 +2810,7 @@
       "additionalProperties": false
     },
     "ModelOutput": {
+      "description": "Output from model generation.",
       "properties": {
         "model": {
           "title": "Model",
@@ -2846,6 +2885,7 @@
       "additionalProperties": false
     },
     "ModelUsage": {
+      "description": "Token usage for completion.",
       "properties": {
         "input_tokens": {
           "default": 0,
@@ -2899,6 +2939,7 @@
       "additionalProperties": false
     },
     "Sample": {
+      "description": "Sample for an evaluation task.",
       "properties": {
         "input": {
           "anyOf": [
@@ -3173,7 +3214,7 @@
       "type": "array"
     },
     "Score": {
-      "description": "Score generated by a scorer.\n\nArgs:\n   value (Value): Score value.\n   answer (str | None): Answer extracted from model output (optional).\n   explanation (str | None): Explanation of score (optional).\n   metadata (dict[str,Any]): Additional metadata related to the score.",
+      "description": "Score generated by a scorer.",
       "properties": {
         "value": {
           "anyOf": [
@@ -3281,7 +3322,7 @@
       "additionalProperties": false
     },
     "ScoreEvent": {
-      "description": "Event with sample score.",
+      "description": "Event with score.\n\nCan be the final score for a `Sample`, or can be an intermediate score\nresulting from a call to `score`.",
       "properties": {
         "timestamp": {
           "format": "date-time",
@@ -3326,6 +3367,11 @@
           ],
           "default": null,
           "title": "Target"
+        },
+        "intermediate": {
+          "default": false,
+          "title": "Intermediate",
+          "type": "boolean"
         }
       },
       "required": [
@@ -3333,7 +3379,8 @@
         "pending",
         "event",
         "score",
-        "target"
+        "target",
+        "intermediate"
       ],
       "title": "ScoreEvent",
       "type": "object",
@@ -4223,6 +4270,7 @@
       "additionalProperties": false
     }
   },
+  "description": "Evaluation log.",
   "properties": {
     "version": {
       "default": 2,
@@ -4244,37 +4292,7 @@
       "$ref": "#/$defs/EvalSpec"
     },
     "plan": {
-      "$ref": "#/$defs/EvalPlan",
-      "default": {
-        "name": "plan",
-        "steps": [],
-        "finish": null,
-        "config": {
-          "best_of": null,
-          "cache_prompt": null,
-          "frequency_penalty": null,
-          "internal_tools": null,
-          "logit_bias": null,
-          "logprobs": null,
-          "max_connections": null,
-          "max_retries": null,
-          "max_tokens": null,
-          "max_tool_output": null,
-          "num_choices": null,
-          "parallel_tool_calls": null,
-          "presence_penalty": null,
-          "reasoning_effort": null,
-          "reasoning_history": null,
-          "seed": null,
-          "stop_seqs": null,
-          "system_message": null,
-          "temperature": null,
-          "timeout": null,
-          "top_k": null,
-          "top_logprobs": null,
-          "top_p": null
-        }
-      }
+      "$ref": "#/$defs/EvalPlan"
     },
     "results": {
       "anyOf": [
@@ -4288,12 +4306,7 @@
       "default": null
     },
     "stats": {
-      "$ref": "#/$defs/EvalStats",
-      "default": {
-        "started_at": "",
-        "completed_at": "",
-        "model_usage": {}
-      }
+      "$ref": "#/$defs/EvalStats"
     },
     "error": {
       "anyOf": [

inspect_ai/_view/www/package.json

@@ -8,7 +8,7 @@
   "scripts": {
     "build": "vite build",
     "watch": "vite build --watch",
-    "dev-watch": "vite build --mode development --watch",
+    "dev-watch": "NODE_ENV=development vite build --mode development --watch",
     "dev": "vite",
     "prettier:check": "prettier --check src",
     "prettier:write": "prettier --write src",

inspect_ai/_view/www/src/App.tsx

@@ -990,6 +990,7 @@ const defaultScorers = (log: EvalSummary): Array<ScorerInfo> => {
       }, [] as Array<ScorerInfo>);
   } else if (log.sampleSummaries && log.sampleSummaries.length > 0) {
     const scores = log.sampleSummaries[0].scores;
+
     if (scores !== null) {
       return Object.keys(scores).map((key) => {
         return {

inspect_ai/_view/www/src/components/AnsiDisplay.tsx

@@ -1,6 +1,6 @@
 import { ANSIColor, ANSIOutput, ANSIOutputRun, ANSIStyle } from "ansi-output";
 import clsx from "clsx";
-import "./ANSIDisplay.css";
+import "./AnsiDisplay.css";
 
 interface ANSIDisplayProps {
   output: string;

inspect_ai/_view/www/src/components/JsonPanel.tsx

@@ -1,7 +1,7 @@
 import clsx from "clsx";
 import { highlightElement } from "prismjs";
 import React, { useEffect, useMemo, useRef } from "react";
-import "./JSONPanel.css";
+import "./JsonPanel.css";
 
 const kPrismRenderMaxSize = 250000;
 

inspect_ai/_view/www/src/components/LargeModal.tsx

@@ -78,54 +78,6 @@ export const LargeModal: React.FC<LargeModalProps> = ({
     [setInitialScrollPosition],
   );
 
-  // Capture header elements
-  const headerEls = [];
-  // The title
-  headerEls.push(
-    <div className={clsx("modal-title", "text-size-smaller", styles.title)}>
-      {title || ""}
-    </div>,
-  );
-
-  // A centered text element with tools to the left and right
-  if (detail) {
-    headerEls.push(
-      <div className={styles.detail}>
-        {detailTools?.left
-          ? detailTools.left.map((tool) => {
-              return <TitleTool {...tool} />;
-            })
-          : ""}
-        <div className={clsx("text-size-smaller", styles.detailText)}>
-          <div>{detail}</div>
-        </div>
-
-        {detailTools?.right
-          ? detailTools.right.map((tool) => {
-              return <TitleTool {...tool} />;
-            })
-          : ""}
-      </div>,
-    );
-  }
-
-  // The close 'x'
-  headerEls.push(
-    <button
-      type="button"
-      className={clsx(
-        "btn",
-        "btn-close-large-dialog",
-        "text-size-larger",
-        styles.close,
-      )}
-      onClick={onHide}
-      aria-label="Close"
-    >
-      <HtmlEntity html={"&times;"} />
-    </button>,
-  );
-
   return (
     <div
       id={id}
@@ -147,7 +99,45 @@ export const LargeModal: React.FC<LargeModalProps> = ({
         role="document"
       >
         <div className={clsx("modal-content", styles.content)}>
-          <div className={clsx("modal-header", styles.header)}>{headerEls}</div>
+          <div className={clsx("modal-header", styles.header)}>
+            <div
+              className={clsx("modal-title", "text-size-smaller", styles.title)}
+            >
+              {title || ""}
+            </div>
+
+            {detail ? (
+              <div className={styles.detail}>
+                {detailTools?.left
+                  ? detailTools.left.map((tool, idx) => {
+                      return <TitleTool key={`tool-left-${idx}`} {...tool} />;
+                    })
+                  : ""}
+                <div className={clsx("text-size-smaller", styles.detailText)}>
+                  <div>{detail}</div>
+                </div>
+
+                {detailTools?.right
+                  ? detailTools.right.map((tool, idx) => {
+                      return <TitleTool key={`tool-right-${idx}`} {...tool} />;
+                    })
+                  : ""}
+              </div>
+            ) : undefined}
+            <button
+              type="button"
+              className={clsx(
+                "btn",
+                "btn-close-large-dialog",
+                "text-size-larger",
+                styles.close,
+              )}
+              onClick={onHide}
+              aria-label="Close"
+            >
+              <HtmlEntity html={"&times;"} />
+            </button>
+          </div>
           <ProgressBar animating={showProgress} />
           <div className={"modal-body"} ref={scrollRef} onScroll={onScroll}>
             {children}

inspect_ai/_view/www/src/components/NavPills.tsx

@@ -26,6 +26,7 @@ export const NavPills: React.FC<NavPillsProps> = ({ children }) => {
         : `Tab ${idx}`;
     return (
       <NavPill
+        key={`nav-pill-contents-${idx}`}
         title={title}
         activeItem={activeItem}
         setActiveItem={setActiveItem}
@@ -34,9 +35,10 @@ export const NavPills: React.FC<NavPillsProps> = ({ children }) => {
   });
 
   // Wrap each of the children in a 'body' to control its visibility
-  const navBodies = children.map((child) => {
+  const navBodies = children.map((child, idx) => {
     return (
       <div
+        key={`nav-pill-container-${idx}`}
         className={
           child["props"]?.title === activeItem ? styles.visible : styles.hidden
         }

inspect_ai/_view/www/src/components/TabSet.tsx

@@ -1,6 +1,8 @@
 import clsx from "clsx";
 import {
+  Children,
   Fragment,
+  isValidElement,
   MouseEvent,
   ReactElement,
   useCallback,
@@ -46,10 +48,7 @@ export const TabSet: React.FC<TabSetProps> = ({
   tools,
   children,
 }) => {
-  const validTabs: ReactElement<TabPanelProps>[] = Array.isArray(children)
-    ? (children.filter(Boolean) as ReactElement<TabPanelProps>[])
-    : [children];
-
+  const validTabs = flattenChildren(children);
   if (validTabs.length === 0) return null;
 
   return (
@@ -198,3 +197,19 @@ const TabTools: React.FC<{ tools?: React.ReactNode }> = ({ tools }) => (
 // Utility functions
 const computeTabId = (id: string, index: number) => `${id}-${index}`;
 const computeTabContentsId = (id: string) => `${id}-contents`;
+
+const flattenChildren = (
+  children: React.ReactNode,
+): ReactElement<TabPanelProps>[] => {
+  return Children.toArray(children).flatMap((child) => {
+    if (isValidElement(child)) {
+      const element = child as React.ReactElement<any>;
+
+      if (element.type === Fragment) {
+        return flattenChildren(element.props.children);
+      }
+      return element;
+    }
+    return [];
+  });
+};

inspect_ai/_view/www/src/logfile/remoteLogFile.ts

@@ -101,7 +101,6 @@ export const openRemoteLogFile = async (
     if (remoteZipFile.centralDirectory.has(sampleFile)) {
       return (await readJSONFile(sampleFile, MAX_BYTES)) as EvalSample;
     } else {
-      console.log({ dir: remoteZipFile.centralDirectory });
       throw new Error(
         `Unable to read sample file ${sampleFile} - it is not present in the manifest.`,
       );