inspect-ai 0.3.55__py3-none-any.whl → 0.3.57__py3-none-any.whl

inspect_ai/_view/www/package.json

@@ -26,6 +26,7 @@
   },
   "dependencies": {
     "@popperjs/core": "^2.11.8",
+    "asciinema-player": "^3.8.1",
     "bootstrap": "^5.3.3",
     "bootstrap-icons": "^1.11.3",
     "clipboard": "^2.0.11",

inspect_ai/_view/www/src/App.mjs

@@ -3,6 +3,7 @@ import "bootstrap-icons/font/bootstrap-icons.css";
 import "prismjs/themes/prism.css";
 import "prismjs";
 import "../App.css";
+import "asciinema-player/dist/bundle/asciinema-player.css";
 
 import { default as ClipboardJS } from "clipboard";
 // @ts-ignore
@@ -31,7 +32,10 @@ import { FindBand } from "./components/FindBand.mjs";
 import { isVscode } from "./utils/Html.mjs";
 import { getVscodeApi } from "./utils/vscode.mjs";
 import { kDefaultSort } from "./constants.mjs";
-import { createsSamplesDescriptor } from "./samples/SamplesDescriptor.mjs";
+import {
+  createEvalDescriptor,
+  createSamplesDescriptor,
+} from "./samples/SamplesDescriptor.mjs";
 import { byEpoch, bySample, sortSamples } from "./samples/tools/SortFilter.mjs";
 import { resolveAttachments } from "./utils/attachments.mjs";
 import { filterFnForType } from "./samples/tools/filters.mjs";
@@ -75,7 +79,7 @@ export function App({
     initialState?.headersLoading || false,
   );
 
-  // Selected Log
+  /** @type {[import("./Types.mjs").CurrentLog, function(import("./Types.mjs").CurrentLog): void]} */
   const [selectedLog, setSelectedLog] = useState(
     initialState?.selectedLog || {
       contents: undefined,
@@ -94,6 +98,7 @@ export function App({
       ? initialState.selectedSampleIndex
       : -1,
   );
+  /** @type {[import("./types/log").EvalSample, function(import("./types/log").EvalSample): void]} */
   const [selectedSample, setSelectedSample] = useState(
     initialState?.selectedSample,
   );
@@ -325,7 +330,7 @@ export function App({
 
     // Set the grouping
     let grouping = "none";
-    if (samplesDescriptor?.epochs > 1) {
+    if (samplesDescriptor?.evalDescriptor?.epochs > 1) {
       if (byEpoch(sort) || epoch !== "all") {
         grouping = "epoch";
       } else if (bySample(sort)) {
@@ -338,14 +343,17 @@ export function App({
     setGroupByOrder(order);
   }, [selectedLog, filter, sort, epoch]);
 
-  const samplesDescriptor = useMemo(() => {
-    return createsSamplesDescriptor(
+  const evalDescriptor = useMemo(() => {
+    return createEvalDescriptor(
       scores,
       selectedLog.contents?.sampleSummaries,
       selectedLog.contents?.eval?.config?.epochs || 1,
-      score,
     );
-  }, [selectedLog, scores, score]);
+  }, [selectedLog, scores]);
+
+  const samplesDescriptor = useMemo(() => {
+    return createSamplesDescriptor(evalDescriptor, score);
+  }, [evalDescriptor, score]);
 
   const refreshSampleTab = useCallback(
     (sample) => {
@@ -512,9 +520,11 @@ export function App({
       // Reset the workspace tab
       const hasSamples =
         !!log.sampleSummaries && log.sampleSummaries.length > 0;
-      const showSamples = log.status !== "error" && hasSamples;
+      const showSamples = hasSamples;
       setSelectedWorkspaceTab(
-        showSamples ? kEvalWorkspaceTabId : kInfoWorkspaceTabId,
+        log.status !== "error" && hasSamples
+          ? kEvalWorkspaceTabId
+          : kInfoWorkspaceTabId,
       );
 
       // Select the default scorer to use

inspect_ai/_view/www/src/Types.mjs

@@ -8,7 +8,6 @@
  * @typedef {Object} CurrentLog
  * @property {string} name
  * @property {import("./api/Types.mjs").EvalSummary} contents
- * @property {string} raw
  */
 
 /**

inspect_ai/_view/www/src/api/Types.mjs

@@ -30,15 +30,26 @@
  * @property { import("../types/log").Input } input
  * @property { import("../types/log").Target } target
  * @property { import("../types/log").Scores1 } scores
+ * @property { string } [error]
  * @property { import("../types/log").Type11 } [limit]
  */
 
 /**
-* @typedef {Object} Capabilities
-* @property {boolean} downloadFiles - Indicates if file downloads are supported.
-* @property {boolean} webWorkers - Indicates if web workers are supported.
-*
+ * Fields shared by EvalSample and SampleSummary.
+ * Contains only fields that are copied verbatim in src/inspect_ai/log/_recorders/eval.py.
+ *
+ * @typedef {Object} BasicSampleData
+ * @property { number | string } id
+ * @property { number } epoch
+ * @property { import("../types/log").Target } target
+ * @property { import("../types/log").Scores1 } scores
+ */
 
+/**
+ * @typedef {Object} Capabilities
+ * @property {boolean} downloadFiles - Indicates if file downloads are supported.
+ * @property {boolean} webWorkers - Indicates if web workers are supported.
+ */
 
 /**
  * @typedef {Object} LogViewAPI

inspect_ai/_view/www/src/api/api-http.mjs

@@ -56,6 +56,7 @@ function simpleHttpAPI(logInfo) {
         });
         return Promise.resolve({
           files: logs,
+          log_dir,
         });
       } else if (log_file) {
         // Check the cache
@@ -76,6 +77,7 @@ function simpleHttpAPI(logInfo) {
 
         return {
           files: [result],
+          log_dir,
         };
       } else {
         // No log.json could be found, and there isn't a log file,

inspect_ai/_view/www/src/appearance/Icons.mjs

@@ -123,6 +123,7 @@
  * @property {string} toggle-right
  * @property {string} more
  * @property {string} next
+ * @property {string} play
  * @property {string} previous
  * @property {string} refresh
  * @property {RoleIcons} role
@@ -209,6 +210,7 @@ export const ApplicationIcons = {
   more: "bi bi-zoom-in",
   "multiple-choice": "bi bi-card-list",
   next: "bi bi-chevron-right",
+  play: "bi bi-play-fill",
   previous: "bi bi-chevron-left",
   refresh: "bi bi-arrow-clockwise",
   role: {

inspect_ai/_view/www/src/components/AsciiCinemaPlayer.mjs

@@ -0,0 +1,74 @@
+// @ts-check
+import { html } from "htm/preact";
+import { useEffect, useRef } from "preact/hooks";
+
+// Import the asciinema player library
+import * as AsciinemaPlayer from "asciinema-player";
+
+/**
+ * Renders the ChatView component.
+ *
+ * @param {Object} props - The properties passed to the component.
+ * @param {string} props.id - The ID for the chat view.
+ * @param {string} props.inputUrl - The input url for the player
+ * @param {string} props.outputUrl - The output url for the player
+ * @param {string} props.timingUrl - The timing url for the player
+ * @param {number} [props.rows] - The rows for the player's initial size
+ * @param {number} [props.cols] - The cols for the player's initial size
+ * @param {string} [props.fit] - how to fit the player
+ * @param {Object} [props.style] - Inline styles for the chat view.
+ * @param {number} [props.speed] - The speed to play (1 = 1x, 1.5 = 1.5x...)
+ * @param {boolean} [props.autoPlay] - Whether to autoplay
+ * @param {boolean} [props.loop] - Whether to loop
+ * @param {string} [props.theme] - The terminal theme (e.g. "solarized-dark")
+ * @param {number} [props.idleTimeLimit] - The amount to compress idle time to
+ * @returns {import("preact").JSX.Element} The component.
+ */
+export const AsciiCinemaPlayer = ({
+  id,
+  rows,
+  cols,
+  inputUrl,
+  outputUrl,
+  timingUrl,
+  fit,
+  speed,
+  autoPlay,
+  loop,
+  theme,
+  idleTimeLimit = 2,
+  style,
+}) => {
+  const playerContainerRef = useRef();
+  useEffect(() => {
+    const player = AsciinemaPlayer.create(
+      {
+        url: [timingUrl, outputUrl, inputUrl],
+        parser: "typescript",
+      },
+      playerContainerRef.current,
+      {
+        rows: rows,
+        cols: cols,
+        autoPlay,
+        loop,
+        theme,
+        speed,
+        idleTimeLimit,
+        fit,
+      },
+    );
+    player.play();
+    return () => {
+      player.dispose();
+    };
+  }, []);
+
+  return html`
+    <div
+      id="asciinema-player-${id || "default"}"
+      ref=${playerContainerRef}
+      style=${{ ...style }}
+    ></div>
+  `;
+};

inspect_ai/_view/www/src/components/CopyButton.mjs

@@ -32,7 +32,6 @@ export const CopyButton = ({ value }) => {
       if (iEl.tagName === "BUTTON") {
         iEl = iEl.firstChild;
       }
-      console.log({ iEl });
       if (iEl) {
         if (iEl) {
           iEl.className = `${ApplicationIcons.confirm} primary`;

inspect_ai/_view/www/src/components/ExpandablePanel.mjs

@@ -14,8 +14,8 @@ export const ExpandablePanel = ({
   const [collapsed, setCollapsed] = useState(collapse);
   const [showToggle, setShowToggle] = useState(false);
 
-  const contentsRef = useRef();
-  const observerRef = useRef();
+  const contentsRef = useRef(/** @type {HTMLElement|null} */ (null));
+  const observerRef = useRef(/** @type {IntersectionObserver|null} */ (null));
 
   // Ensure that when content changes, we reset the collapse state.
   useEffect(() => {

inspect_ai/_view/www/src/components/FindBand.mjs

@@ -4,7 +4,7 @@ import { ApplicationIcons } from "../appearance/Icons.mjs";
 import { FontSize } from "../appearance/Fonts.mjs";
 
 export const FindBand = ({ hideBand }) => {
-  const searchBoxRef = useRef();
+  const searchBoxRef = useRef(/** @type {HTMLInputElement|null} */ (null));
   useEffect(() => {
     searchBoxRef.current.focus();
   }, []);
@@ -31,13 +31,14 @@ export const FindBand = ({ hideBand }) => {
     };
 
     // capture what is focused
-    const focusedElement = document.activeElement;
+    const focusedElement = /** @type {HTMLElement} */ (document.activeElement);
+    // @ts-expect-error: `Window.find` is non-standard
     const result = window.find(term, false, !!back, false, false, true, false);
     const noResultEl = window.document.getElementById(
       "inspect-find-no-results",
     );
     if (result) {
-      noResultEl.style.opacity = 0;
+      noResultEl.style.opacity = "0";
       const selection = window.getSelection();
       if (selection.rangeCount > 0) {
         // See if the parent is an expandable panel and expand it
@@ -58,7 +59,7 @@ export const FindBand = ({ hideBand }) => {
         }, 100);
       }
     } else {
-      noResultEl.style.opacity = 1;
+      noResultEl.style.opacity = "1";
     }
 
     // Return focus to the previously focused element

inspect_ai/_view/www/src/components/HumanBaselineView.mjs

@@ -0,0 +1,168 @@
+// @ts-check
+import { html } from "htm/preact";
+import { useEffect } from "preact/hooks";
+import { formatDateTime, formatTime } from "../utils/Format.mjs";
+import { AsciiCinemaPlayer } from "./AsciiCinemaPlayer.mjs";
+import { TextStyle } from "../appearance/Fonts.mjs";
+import { LightboxCarousel } from "./LightboxCarousel.mjs";
+
+/**
+ * @typedef {Object} SessionLog
+ * @property {string} name - The name of this session
+ * @property {string} user - The user for this session
+ * @property {string} input - The input for this session
+ * @property {string} output - The output for this session
+ * @property {string} timing - The timing for this session
+ */
+
+/**
+ * Renders the HumanBaselineView component.
+ *
+ * @param {Object} props - The properties passed to the component.
+ * @param {Date} props.started - When the baselining started
+ * @param {boolean} props.running - Whether the baselining is running
+ * @param {boolean} [props.completed] - Whether the baselining was completed
+ * @param {number} [props.runtime] - Duration of baselining in seconds
+ * @param {string} [props.answer] - The answer for the baselining
+ * @param {SessionLog[]} [props.sessionLogs] - The session logs for the baselining
+ * @returns {import("preact").JSX.Element} The component.
+ */
+export const HumanBaselineView = ({
+  started,
+  runtime,
+  answer,
+  completed,
+  running,
+  sessionLogs,
+}) => {
+  const player_fns = [];
+
+  // handle creation and revoking of these URLs
+  const revokableUrls = [];
+  const revokableUrl = (data) => {
+    const blob = new Blob([data], { type: "text/plain" });
+    const url = URL.createObjectURL(blob);
+    revokableUrls.push(url);
+    return url;
+  };
+
+  useEffect(() => {
+    return () => {
+      revokableUrls.forEach((url) => URL.revokeObjectURL(url));
+    };
+  }, []);
+
+  // Make a player for each session log
+  let count = 1;
+  let maxCols = 0;
+
+  for (const sessionLog of sessionLogs) {
+    const rows = extractSize(sessionLog.output, "LINES");
+    const cols = extractSize(sessionLog.output, "COLUMNS");
+    maxCols = Math.max(maxCols, parseInt(cols));
+
+    const currentCount = count;
+    const title =
+      sessionLogs.length === 1
+        ? "Terminal Session"
+        : `Terminal Session ${currentCount}`;
+
+    player_fns.push({
+      label: title,
+      render: () => html`
+        <${AsciiCinemaPlayer}
+          id=${`player-${currentCount}`}
+          inputUrl=${revokableUrl(sessionLog.input)}
+          outputUrl=${revokableUrl(sessionLog.output)}
+          timingUrl=${revokableUrl(sessionLog.timing)}
+          rows=${rows}
+          cols=${cols}
+          style=${{
+            maxHeight: "100vh",
+            maxWidth: "100vw",
+            height: `${parseInt(rows) * 2}em`,
+            width: `${parseInt(cols) * 2}em`,
+          }}
+          fit="both"
+        />
+      `,
+    });
+    count += 1;
+  }
+
+  const StatusMessage = ({ completed, running, answer }) => {
+    if (running) {
+      return html`<span style=${{ ...TextStyle.label }}>Running</span>`;
+    } else if (completed) {
+      return html`<span style=${{ ...TextStyle.label, marginRight: "0.5em" }}
+          >Answer</span
+        ><span>${answer}</span>`;
+    } else {
+      return "Unknown status";
+    }
+  };
+
+  return html`<div style=${{ display: "flex", justifyContent: "center" }}>
+    <div
+      style=${{
+        display: "grid",
+        gridTemplateColumns: "1fr 1fr 1fr",
+        gridTemplateRows: "auto auto",
+        width: "100%",
+      }}
+    >
+      <div
+        style=${{
+          justifySelf: "start",
+          ...TextStyle.label,
+        }}
+      >
+        ${started ? formatDateTime(started) : ""}${runtime
+          ? ` (${formatTime(Math.floor(runtime))})`
+          : ""}
+      </div>
+      <div
+        style=${{
+          justifySelf: "center",
+          ...TextStyle.label,
+        }}
+      ></div>
+      <div
+        style=${{
+          justifySelf: "end",
+        }}
+      >
+        <${StatusMessage}
+          completed=${completed}
+          running=${running}
+          answer=${answer}
+        />
+      </div>
+      <div
+        style=${{
+          gridColumn: "span 3",
+          width: "100%",
+        }}
+      >
+        <${LightboxCarousel} slides=${player_fns} />
+      </div>
+    </div>
+  </div>`;
+};
+
+/**
+ * Extracts a numeric size value from a string based on a given label.
+ *
+ * Searches the input string for a pattern matching the format `LABEL="VALUE"`,
+ * where `LABEL` is the provided label and `VALUE` is a numeric value.
+ *
+ * @param {string} value - The input string to search within.
+ * @param {string} label - The label to look for in the string.
+ * @returns {string | undefined} The extracted size as a string if found, otherwise `undefined`.
+ */
+const extractSize = (value, label) => {
+  const regex = new RegExp(`${label}="(\\d+)"`);
+  const match = value.match(regex);
+  const size = match ? match[1] : undefined;
+  return size;
+};

inspect_ai/_view/www/src/components/LargeModal.mjs

@@ -31,7 +31,7 @@ export const LargeModal = (props) => {
 
   // Support restoring the scroll position
   // but only do this for the first time that the children are set
-  const scrollRef = useRef();
+  const scrollRef = useRef(/** @type {HTMLElement|null} */ (null));
   useEffect(() => {
     if (scrollRef.current) {
       setTimeout(() => {

inspect_ai/_view/www/src/components/LightboxCarousel.mjs

@@ -0,0 +1,217 @@
+import { html } from "htm/preact";
+import { useState, useCallback, useEffect } from "preact/hooks";
+import { ApplicationIcons } from "../appearance/Icons.mjs";
+
+/**
+ * @typedef {Object} Slide
+ * @property {string} label - The label for the slide.
+ * @property {() => import("preact").JSX.Element} render - A function that returns another function to render the slide as a JSX element.
+ */
+
+/**
+ * LightboxCarousel component provides a carousel with lightbox functionality.
+ * @param {Object} props - Component properties.
+ * @property {Array<Slide>} props.slides - Array of slide render functions.
+ * @returns {import("preact").JSX.Element} LightboxCarousel component.
+ */
+export const LightboxCarousel = ({ slides }) => {
+  const [isOpen, setIsOpen] = useState(false);
+  const [showOverlay, setShowOverlay] = useState(false);
+  const [currentIndex, setCurrentIndex] = useState(0);
+
+  const openLightbox = (index) => {
+    setCurrentIndex(index);
+    setShowOverlay(true);
+
+    // Slight delay before setting isOpen so the fade-in starts from opacity: 0
+    setTimeout(() => setIsOpen(true), 10);
+  };
+
+  const closeLightbox = () => {
+    setIsOpen(false);
+  };
+
+  // Remove the overlay from the DOM after fade-out completes
+  useEffect(() => {
+    if (!isOpen && showOverlay) {
+      const timer = setTimeout(() => {
+        setShowOverlay(false);
+      }, 300); // match your transition duration
+      return () => clearTimeout(timer);
+    }
+  }, [isOpen, showOverlay]);
+
+  const showNext = useCallback(() => {
+    setCurrentIndex((prev) => {
+      return (prev + 1) % slides.length;
+    });
+  }, [slides]);
+
+  const showPrev = useCallback(() => {
+    setCurrentIndex((prev) => (prev - 1 + slides.length) % slides.length);
+  }, [slides]);
+
+  // Keyboard Navigation
+  useEffect(() => {
+    if (!isOpen) return;
+    const handleKeyUp = (e) => {
+      if (e.key === "Escape") {
+        closeLightbox();
+      } else if (e.key === "ArrowRight") {
+        showNext();
+      } else if (e.key === "ArrowLeft") {
+        showPrev();
+      }
+      e.preventDefault();
+      e.stopPropagation();
+    };
+    window.addEventListener("keyup", handleKeyUp, true);
+    return () => window.removeEventListener("keyup", handleKeyUp);
+  }, [isOpen, showNext, showPrev]);
+
+  // Common button style
+  const buttonStyle = {
+    position: "absolute",
+    top: "50%",
+    transform: "translateY(-50%)",
+    background: "none",
+    color: "#fff",
+    border: "none",
+    padding: "0.5em",
+    fontSize: "3em",
+    cursor: "pointer",
+    zIndex: "9999",
+  };
+
+  const prevButtonStyle = {
+    ...buttonStyle,
+    left: "10px",
+  };
+
+  const nextButtonStyle = {
+    ...buttonStyle,
+    right: "10px",
+  };
+
+  // Overlay style (fixed to fill the screen, flex for centering)
+  const overlayStyle = {
+    position: "fixed",
+    top: 0,
+    left: 0,
+    width: "100vw",
+    height: "100vh",
+    background: "rgba(0,0,0,0.8)",
+    display: "flex",
+    alignItems: "center",
+    justifyContent: "center",
+    opacity: isOpen ? "1" : "0",
+    visibility: isOpen ? "visible" : "hidden",
+    transition: "opacity 0.3s ease, visibility 0.3s ease",
+    zIndex: 9998,
+  };
+
+  // Close button style
+  const closeButtonWrapperStyle = {
+    position: "absolute",
+    top: "10px",
+    right: "10px",
+  };
+
+  const closeButtonStyle = {
+    border: "none",
+    background: "none",
+    color: "#fff",
+    fontSize: "3em",
+    fontWeight: "500",
+    cursor: "pointer",
+    paddingLeft: "1em",
+    paddingBottom: "1em",
+    zIndex: "10000",
+  };
+
+  // Lightbox content container style
+  const contentStyle = {
+    maxWidth: "90%",
+    maxHeight: "90%",
+    display: "flex",
+    alignItems: "center",
+    justifyContent: "center",
+    position: "relative",
+    // fade in/out
+    opacity: isOpen ? "1" : "0",
+    visibility: isOpen ? "visible" : "hidden",
+    transition: "opacity 0.3s ease, visibility 0.3s ease",
+    zIndex: 9999,
+  };
+
+  return html`
+    <div className="lightbox-carousel-container">
+      <!-- Thumbnails -->
+      <div
+        className="carousel-thumbs"
+        style=${{
+          display: "grid",
+          gridTemplateColumns: "auto auto auto auto",
+        }}
+      >
+        ${slides.map((slide, index) => {
+          return html`
+            <div
+              key=${index}
+              className="carousel-thumb"
+              onClick=${() => openLightbox(index)}
+              style=${{
+                background: "black",
+                color: "white",
+                padding: "4em 0",
+                border: "0",
+                margin: "5px",
+                cursor: "pointer",
+                textAlign: "center",
+              }}
+            >
+              <div>${slide.label}</div>
+              <div>
+                <i
+                  class=${ApplicationIcons.play}
+                  style=${{ fontSize: "4em" }}
+                ></i>
+              </div>
+            </div>
+          `;
+        })}
+      </div>
+
+      <!-- Lightbox Overlay -->
+      ${showOverlay &&
+      html`
+        <div className="lightbox-overlay" style=${overlayStyle}>
+          <div style=${closeButtonWrapperStyle}>
+            <button style=${closeButtonStyle} onClick=${closeLightbox}>
+              <i class=${ApplicationIcons.close}></i>
+            </button>
+          </div>
+
+          ${slides.length > 1
+            ? html` <button style=${prevButtonStyle} onClick=${showPrev}>
+                <i class=${ApplicationIcons.previous}></i>
+              </button>`
+            : ""}
+          ${slides.length > 1
+            ? html` <button style=${nextButtonStyle} onClick=${showNext}>
+                <i class=${ApplicationIcons.next}></i>
+              </button>`
+            : ""}
+
+          <div
+            key=${`carousel-slide-${currentIndex}`}
+            className="lightbox-content"
+            style=${contentStyle}
+          >
+            ${slides[currentIndex].render()}
+          </div>
+        </div>
+      `}
+    </div>
+  `;
+};