inspect-ai 0.3.56__py3-none-any.whl → 0.3.58__py3-none-any.whl

inspect_ai/_view/www/log-schema.json

@@ -210,6 +210,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -281,6 +287,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -336,6 +348,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -429,6 +447,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },
@@ -482,6 +506,36 @@
       "type": "object",
       "additionalProperties": false
     },
+    "ContentAudio": {
+      "properties": {
+        "type": {
+          "const": "audio",
+          "default": "audio",
+          "title": "Type",
+          "type": "string"
+        },
+        "audio": {
+          "title": "Audio",
+          "type": "string"
+        },
+        "format": {
+          "enum": [
+            "wav",
+            "mp3"
+          ],
+          "title": "Format",
+          "type": "string"
+        }
+      },
+      "required": [
+        "type",
+        "audio",
+        "format"
+      ],
+      "title": "ContentAudio",
+      "type": "object",
+      "additionalProperties": false
+    },
     "ContentImage": {
       "properties": {
         "type": {
@@ -535,6 +589,37 @@
       "type": "object",
       "additionalProperties": false
     },
+    "ContentVideo": {
+      "properties": {
+        "type": {
+          "const": "video",
+          "default": "video",
+          "title": "Type",
+          "type": "string"
+        },
+        "video": {
+          "title": "Video",
+          "type": "string"
+        },
+        "format": {
+          "enum": [
+            "mp4",
+            "mpeg",
+            "mov"
+          ],
+          "title": "Format",
+          "type": "string"
+        }
+      },
+      "required": [
+        "type",
+        "video",
+        "format"
+      ],
+      "title": "ContentVideo",
+      "type": "object",
+      "additionalProperties": false
+    },
     "ErrorEvent": {
       "description": "Event with sample error.",
       "properties": {
@@ -657,18 +742,6 @@
           "default": null,
           "title": "Epochs Reducer"
         },
-        "trace": {
-          "anyOf": [
-            {
-              "type": "boolean"
-            },
-            {
-              "type": "null"
-            }
-          ],
-          "default": null,
-          "title": "Trace"
-        },
         "approval": {
           "anyOf": [
             {
@@ -847,7 +920,6 @@
         "sample_id",
         "epochs",
         "epochs_reducer",
-        "trace",
         "approval",
         "fail_on_error",
         "message_limit",
@@ -1482,7 +1554,7 @@
         },
         "samples": {
           "items": {
-            "$ref": "#/$defs/SampleScore"
+            "$ref": "#/$defs/EvalSampleScore"
           },
           "title": "Samples",
           "type": "array"
@@ -1497,6 +1569,129 @@
       "type": "object",
       "additionalProperties": false
     },
+    "EvalSampleScore": {
+      "properties": {
+        "value": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "integer"
+            },
+            {
+              "type": "number"
+            },
+            {
+              "type": "boolean"
+            },
+            {
+              "items": {
+                "anyOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "integer"
+                  },
+                  {
+                    "type": "number"
+                  },
+                  {
+                    "type": "boolean"
+                  }
+                ]
+              },
+              "type": "array"
+            },
+            {
+              "additionalProperties": {
+                "anyOf": [
+                  {
+                    "type": "string"
+                  },
+                  {
+                    "type": "integer"
+                  },
+                  {
+                    "type": "number"
+                  },
+                  {
+                    "type": "boolean"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ]
+              },
+              "type": "object"
+            }
+          ],
+          "title": "Value"
+        },
+        "answer": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Answer"
+        },
+        "explanation": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Explanation"
+        },
+        "metadata": {
+          "anyOf": [
+            {
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Metadata"
+        },
+        "sample_id": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "integer"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Sample Id"
+        }
+      },
+      "required": [
+        "value",
+        "answer",
+        "explanation",
+        "metadata",
+        "sample_id"
+      ],
+      "title": "EvalSampleScore",
+      "type": "object",
+      "additionalProperties": false
+    },
     "EvalScore": {
       "properties": {
         "name": {
@@ -2905,130 +3100,6 @@
       "type": "object",
       "additionalProperties": false
     },
-    "SampleScore": {
-      "description": "Score for a Sample\n\nArgs:\n   sample_id: (str | int | None) Unique id of a sample",
-      "properties": {
-        "value": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "integer"
-            },
-            {
-              "type": "number"
-            },
-            {
-              "type": "boolean"
-            },
-            {
-              "items": {
-                "anyOf": [
-                  {
-                    "type": "string"
-                  },
-                  {
-                    "type": "integer"
-                  },
-                  {
-                    "type": "number"
-                  },
-                  {
-                    "type": "boolean"
-                  }
-                ]
-              },
-              "type": "array"
-            },
-            {
-              "additionalProperties": {
-                "anyOf": [
-                  {
-                    "type": "string"
-                  },
-                  {
-                    "type": "integer"
-                  },
-                  {
-                    "type": "number"
-                  },
-                  {
-                    "type": "boolean"
-                  },
-                  {
-                    "type": "null"
-                  }
-                ]
-              },
-              "type": "object"
-            }
-          ],
-          "title": "Value"
-        },
-        "answer": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "null"
-            }
-          ],
-          "default": null,
-          "title": "Answer"
-        },
-        "explanation": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "null"
-            }
-          ],
-          "default": null,
-          "title": "Explanation"
-        },
-        "metadata": {
-          "anyOf": [
-            {
-              "type": "object"
-            },
-            {
-              "type": "null"
-            }
-          ],
-          "default": null,
-          "title": "Metadata"
-        },
-        "sample_id": {
-          "anyOf": [
-            {
-              "type": "string"
-            },
-            {
-              "type": "integer"
-            },
-            {
-              "type": "null"
-            }
-          ],
-          "default": null,
-          "title": "Sample Id"
-        }
-      },
-      "required": [
-        "value",
-        "answer",
-        "explanation",
-        "metadata",
-        "sample_id"
-      ],
-      "title": "SampleScore",
-      "type": "object",
-      "additionalProperties": false
-    },
     "SandboxEnvironmentSpec": {
       "maxItems": 2,
       "minItems": 1,
@@ -3722,6 +3793,12 @@
             {
               "$ref": "#/$defs/ContentImage"
             },
+            {
+              "$ref": "#/$defs/ContentAudio"
+            },
+            {
+              "$ref": "#/$defs/ContentVideo"
+            },
             {
               "items": {
                 "anyOf": [
@@ -3730,6 +3807,12 @@
                   },
                   {
                     "$ref": "#/$defs/ContentImage"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentAudio"
+                  },
+                  {
+                    "$ref": "#/$defs/ContentVideo"
                   }
                 ]
               },

inspect_ai/_view/www/src/App.mjs

@@ -32,7 +32,10 @@ import { FindBand } from "./components/FindBand.mjs";
 import { isVscode } from "./utils/Html.mjs";
 import { getVscodeApi } from "./utils/vscode.mjs";
 import { kDefaultSort } from "./constants.mjs";
-import { createsSamplesDescriptor } from "./samples/SamplesDescriptor.mjs";
+import {
+  createEvalDescriptor,
+  createSamplesDescriptor,
+} from "./samples/SamplesDescriptor.mjs";
 import { byEpoch, bySample, sortSamples } from "./samples/tools/SortFilter.mjs";
 import { resolveAttachments } from "./utils/attachments.mjs";
 import { filterFnForType } from "./samples/tools/filters.mjs";
@@ -76,7 +79,7 @@ export function App({
     initialState?.headersLoading || false,
   );
 
-  // Selected Log
+  /** @type {[import("./Types.mjs").CurrentLog, function(import("./Types.mjs").CurrentLog): void]} */
   const [selectedLog, setSelectedLog] = useState(
     initialState?.selectedLog || {
       contents: undefined,
@@ -95,6 +98,7 @@ export function App({
       ? initialState.selectedSampleIndex
       : -1,
   );
+  /** @type {[import("./types/log").EvalSample, function(import("./types/log").EvalSample): void]} */
   const [selectedSample, setSelectedSample] = useState(
     initialState?.selectedSample,
   );
@@ -326,7 +330,7 @@ export function App({
 
     // Set the grouping
     let grouping = "none";
-    if (samplesDescriptor?.epochs > 1) {
+    if (samplesDescriptor?.evalDescriptor?.epochs > 1) {
       if (byEpoch(sort) || epoch !== "all") {
         grouping = "epoch";
       } else if (bySample(sort)) {
@@ -339,14 +343,17 @@ export function App({
     setGroupByOrder(order);
   }, [selectedLog, filter, sort, epoch]);
 
-  const samplesDescriptor = useMemo(() => {
-    return createsSamplesDescriptor(
+  const evalDescriptor = useMemo(() => {
+    return createEvalDescriptor(
       scores,
       selectedLog.contents?.sampleSummaries,
       selectedLog.contents?.eval?.config?.epochs || 1,
-      score,
     );
-  }, [selectedLog, scores, score]);
+  }, [selectedLog, scores]);
+
+  const samplesDescriptor = useMemo(() => {
+    return createSamplesDescriptor(evalDescriptor, score);
+  }, [evalDescriptor, score]);
 
   const refreshSampleTab = useCallback(
     (sample) => {
@@ -513,9 +520,11 @@ export function App({
       // Reset the workspace tab
       const hasSamples =
         !!log.sampleSummaries && log.sampleSummaries.length > 0;
-      const showSamples = log.status !== "error" && hasSamples;
+      const showSamples = hasSamples;
       setSelectedWorkspaceTab(
-        showSamples ? kEvalWorkspaceTabId : kInfoWorkspaceTabId,
+        log.status !== "error" && hasSamples
+          ? kEvalWorkspaceTabId
+          : kInfoWorkspaceTabId,
       );
 
       // Select the default scorer to use

inspect_ai/_view/www/src/Types.mjs

@@ -8,7 +8,6 @@
  * @typedef {Object} CurrentLog
  * @property {string} name
  * @property {import("./api/Types.mjs").EvalSummary} contents
- * @property {string} raw
  */
 
 /**

inspect_ai/_view/www/src/api/Types.mjs

@@ -30,15 +30,26 @@
  * @property { import("../types/log").Input } input
  * @property { import("../types/log").Target } target
  * @property { import("../types/log").Scores1 } scores
+ * @property { string } [error]
  * @property { import("../types/log").Type11 } [limit]
  */
 
 /**
-* @typedef {Object} Capabilities
-* @property {boolean} downloadFiles - Indicates if file downloads are supported.
-* @property {boolean} webWorkers - Indicates if web workers are supported.
-*
+ * Fields shared by EvalSample and SampleSummary.
+ * Contains only fields that are copied verbatim in src/inspect_ai/log/_recorders/eval.py.
+ *
+ * @typedef {Object} BasicSampleData
+ * @property { number | string } id
+ * @property { number } epoch
+ * @property { import("../types/log").Target } target
+ * @property { import("../types/log").Scores1 } scores
+ */
 
+/**
+ * @typedef {Object} Capabilities
+ * @property {boolean} downloadFiles - Indicates if file downloads are supported.
+ * @property {boolean} webWorkers - Indicates if web workers are supported.
+ */
 
 /**
  * @typedef {Object} LogViewAPI

inspect_ai/_view/www/src/api/api-http.mjs

@@ -56,6 +56,7 @@ function simpleHttpAPI(logInfo) {
         });
         return Promise.resolve({
           files: logs,
+          log_dir,
         });
       } else if (log_file) {
         // Check the cache
@@ -76,6 +77,7 @@ function simpleHttpAPI(logInfo) {
 
         return {
           files: [result],
+          log_dir,
         };
       } else {
         // No log.json could be found, and there isn't a log file,

inspect_ai/_view/www/src/components/ExpandablePanel.mjs

@@ -14,8 +14,8 @@ export const ExpandablePanel = ({
   const [collapsed, setCollapsed] = useState(collapse);
   const [showToggle, setShowToggle] = useState(false);
 
-  const contentsRef = useRef();
-  const observerRef = useRef();
+  const contentsRef = useRef(/** @type {HTMLElement|null} */ (null));
+  const observerRef = useRef(/** @type {IntersectionObserver|null} */ (null));
 
   // Ensure that when content changes, we reset the collapse state.
   useEffect(() => {

inspect_ai/_view/www/src/components/FindBand.mjs

@@ -4,7 +4,7 @@ import { ApplicationIcons } from "../appearance/Icons.mjs";
 import { FontSize } from "../appearance/Fonts.mjs";
 
 export const FindBand = ({ hideBand }) => {
-  const searchBoxRef = useRef();
+  const searchBoxRef = useRef(/** @type {HTMLInputElement|null} */ (null));
   useEffect(() => {
     searchBoxRef.current.focus();
   }, []);
@@ -31,13 +31,14 @@ export const FindBand = ({ hideBand }) => {
     };
 
     // capture what is focused
-    const focusedElement = document.activeElement;
+    const focusedElement = /** @type {HTMLElement} */ (document.activeElement);
+    // @ts-expect-error: `Window.find` is non-standard
     const result = window.find(term, false, !!back, false, false, true, false);
     const noResultEl = window.document.getElementById(
       "inspect-find-no-results",
     );
     if (result) {
-      noResultEl.style.opacity = 0;
+      noResultEl.style.opacity = "0";
       const selection = window.getSelection();
       if (selection.rangeCount > 0) {
         // See if the parent is an expandable panel and expand it
@@ -58,7 +59,7 @@ export const FindBand = ({ hideBand }) => {
         }, 100);
       }
     } else {
-      noResultEl.style.opacity = 1;
+      noResultEl.style.opacity = "1";
     }
 
     // Return focus to the previously focused element

inspect_ai/_view/www/src/components/LargeModal.mjs

@@ -31,7 +31,7 @@ export const LargeModal = (props) => {
 
   // Support restoring the scroll position
   // but only do this for the first time that the children are set
-  const scrollRef = useRef();
+  const scrollRef = useRef(/** @type {HTMLElement|null} */ (null));
   useEffect(() => {
     if (scrollRef.current) {
       setTimeout(() => {

inspect_ai/_view/www/src/components/MessageBand.mjs

@@ -8,7 +8,7 @@ export const MessageBand = ({ message, hidden, setHidden, type }) => {
   const bgColor =
     type === "info" ? "var(--bs-light)" : "var(--bs-" + type + "-bg-subtle)";
   const color =
-    "var(--bs-" + (type === "info" ? "secondary" : type) + "-text-emphasis)";
+    type === "info" ? undefined : "var(--bs-" + type + "-text-emphasis)";
 
   return html`
     <div
@@ -32,7 +32,7 @@ export const MessageBand = ({ message, hidden, setHidden, type }) => {
           fontSize: FontSize["title-secondary"],
           margin: "0",
           padding: "0",
-          color: "var(--bs-" + type + "-text-emphasis)",
+          color: color,
           height: FontSize["title-secondary"],
           lineHeight: FontSize["title-secondary"],
         }}