inspect-ai 0.3.55__py3-none-any.whl → 0.3.56__py3-none-any.whl

inspect_ai/_view/www/src/components/HumanBaselineView.mjs

@@ -0,0 +1,168 @@
+// @ts-check
+import { html } from "htm/preact";
+import { useEffect } from "preact/hooks";
+import { formatDateTime, formatTime } from "../utils/Format.mjs";
+import { AsciiCinemaPlayer } from "./AsciiCinemaPlayer.mjs";
+import { TextStyle } from "../appearance/Fonts.mjs";
+import { LightboxCarousel } from "./LightboxCarousel.mjs";
+
+/**
+ * @typedef {Object} SessionLog
+ * @property {string} name - The name of this session
+ * @property {string} user - The user for this session
+ * @property {string} input - The input for this session
+ * @property {string} output - The output for this session
+ * @property {string} timing - The timing for this session
+ */
+
+/**
+ * Renders the HumanBaselineView component.
+ *
+ * @param {Object} props - The properties passed to the component.
+ * @param {Date} props.started - When the baselining started
+ * @param {boolean} props.running - Whether the baselining is running
+ * @param {boolean} [props.completed] - Whether the baselining was completed
+ * @param {number} [props.runtime] - Duration of baselining in seconds
+ * @param {string} [props.answer] - The answer for the baselining
+ * @param {SessionLog[]} [props.sessionLogs] - The session logs for the baselining
+ * @returns {import("preact").JSX.Element} The component.
+ */
+export const HumanBaselineView = ({
+  started,
+  runtime,
+  answer,
+  completed,
+  running,
+  sessionLogs,
+}) => {
+  const player_fns = [];
+
+  // handle creation and revoking of these URLs
+  const revokableUrls = [];
+  const revokableUrl = (data) => {
+    const blob = new Blob([data], { type: "text/plain" });
+    const url = URL.createObjectURL(blob);
+    revokableUrls.push(url);
+    return url;
+  };
+
+  useEffect(() => {
+    return () => {
+      revokableUrls.forEach((url) => URL.revokeObjectURL(url));
+    };
+  }, []);
+
+  // Make a player for each session log
+  let count = 1;
+  let maxCols = 0;
+
+  for (const sessionLog of sessionLogs) {
+    const rows = extractSize(sessionLog.output, "LINES");
+    const cols = extractSize(sessionLog.output, "COLUMNS");
+    maxCols = Math.max(maxCols, parseInt(cols));
+
+    const currentCount = count;
+    const title =
+      sessionLogs.length === 1
+        ? "Terminal Session"
+        : `Terminal Session ${currentCount}`;
+
+    player_fns.push({
+      label: title,
+      render: () => html`
+        <${AsciiCinemaPlayer}
+          id=${`player-${currentCount}`}
+          inputUrl=${revokableUrl(sessionLog.input)}
+          outputUrl=${revokableUrl(sessionLog.output)}
+          timingUrl=${revokableUrl(sessionLog.timing)}
+          rows=${rows}
+          cols=${cols}
+          style=${{
+            maxHeight: "100vh",
+            maxWidth: "100vw",
+            height: `${parseInt(rows) * 2}em`,
+            width: `${parseInt(cols) * 2}em`,
+          }}
+          fit="both"
+        />
+      `,
+    });
+    count += 1;
+  }
+
+  const StatusMessage = ({ completed, running, answer }) => {
+    if (running) {
+      return html`<span style=${{ ...TextStyle.label }}>Running</span>`;
+    } else if (completed) {
+      return html`<span style=${{ ...TextStyle.label, marginRight: "0.5em" }}
+          >Answer</span
+        ><span>${answer}</span>`;
+    } else {
+      return "Unknown status";
+    }
+  };
+
+  return html`<div style=${{ display: "flex", justifyContent: "center" }}>
+    <div
+      style=${{
+        display: "grid",
+        gridTemplateColumns: "1fr 1fr 1fr",
+        gridTemplateRows: "auto auto",
+        width: "100%",
+      }}
+    >
+      <div
+        style=${{
+          justifySelf: "start",
+          ...TextStyle.label,
+        }}
+      >
+        ${started ? formatDateTime(started) : ""}${runtime
+          ? ` (${formatTime(Math.floor(runtime))})`
+          : ""}
+      </div>
+      <div
+        style=${{
+          justifySelf: "center",
+          ...TextStyle.label,
+        }}
+      ></div>
+      <div
+        style=${{
+          justifySelf: "end",
+        }}
+      >
+        <${StatusMessage}
+          completed=${completed}
+          running=${running}
+          answer=${answer}
+        />
+      </div>
+      <div
+        style=${{
+          gridColumn: "span 3",
+          width: "100%",
+        }}
+      >
+        <${LightboxCarousel} slides=${player_fns} />
+      </div>
+    </div>
+  </div>`;
+};
+
+/**
+ * Extracts a numeric size value from a string based on a given label.
+ *
+ * Searches the input string for a pattern matching the format `LABEL="VALUE"`,
+ * where `LABEL` is the provided label and `VALUE` is a numeric value.
+ *
+ * @param {string} value - The input string to search within.
+ * @param {string} label - The label to look for in the string.
+ * @returns {string | undefined} The extracted size as a string if found, otherwise `undefined`.
+ */
+const extractSize = (value, label) => {
+  const regex = new RegExp(`${label}="(\\d+)"`);
+  const match = value.match(regex);
+  const size = match ? match[1] : undefined;
+  return size;
+};

inspect_ai/_view/www/src/components/LightboxCarousel.mjs

@@ -0,0 +1,217 @@
+import { html } from "htm/preact";
+import { useState, useCallback, useEffect } from "preact/hooks";
+import { ApplicationIcons } from "../appearance/Icons.mjs";
+
+/**
+ * @typedef {Object} Slide
+ * @property {string} label - The label for the slide.
+ * @property {() => import("preact").JSX.Element} render - A function that returns another function to render the slide as a JSX element.
+ */
+
+/**
+ * LightboxCarousel component provides a carousel with lightbox functionality.
+ * @param {Object} props - Component properties.
+ * @property {Array<Slide>} props.slides - Array of slide render functions.
+ * @returns {import("preact").JSX.Element} LightboxCarousel component.
+ */
+export const LightboxCarousel = ({ slides }) => {
+  const [isOpen, setIsOpen] = useState(false);
+  const [showOverlay, setShowOverlay] = useState(false);
+  const [currentIndex, setCurrentIndex] = useState(0);
+
+  const openLightbox = (index) => {
+    setCurrentIndex(index);
+    setShowOverlay(true);
+
+    // Slight delay before setting isOpen so the fade-in starts from opacity: 0
+    setTimeout(() => setIsOpen(true), 10);
+  };
+
+  const closeLightbox = () => {
+    setIsOpen(false);
+  };
+
+  // Remove the overlay from the DOM after fade-out completes
+  useEffect(() => {
+    if (!isOpen && showOverlay) {
+      const timer = setTimeout(() => {
+        setShowOverlay(false);
+      }, 300); // match your transition duration
+      return () => clearTimeout(timer);
+    }
+  }, [isOpen, showOverlay]);
+
+  const showNext = useCallback(() => {
+    setCurrentIndex((prev) => {
+      return (prev + 1) % slides.length;
+    });
+  }, [slides]);
+
+  const showPrev = useCallback(() => {
+    setCurrentIndex((prev) => (prev - 1 + slides.length) % slides.length);
+  }, [slides]);
+
+  // Keyboard Navigation
+  useEffect(() => {
+    if (!isOpen) return;
+    const handleKeyUp = (e) => {
+      if (e.key === "Escape") {
+        closeLightbox();
+      } else if (e.key === "ArrowRight") {
+        showNext();
+      } else if (e.key === "ArrowLeft") {
+        showPrev();
+      }
+      e.preventDefault();
+      e.stopPropagation();
+    };
+    window.addEventListener("keyup", handleKeyUp, true);
+    return () => window.removeEventListener("keyup", handleKeyUp);
+  }, [isOpen, showNext, showPrev]);
+
+  // Common button style
+  const buttonStyle = {
+    position: "absolute",
+    top: "50%",
+    transform: "translateY(-50%)",
+    background: "none",
+    color: "#fff",
+    border: "none",
+    padding: "0.5em",
+    fontSize: "3em",
+    cursor: "pointer",
+    zIndex: "9999",
+  };
+
+  const prevButtonStyle = {
+    ...buttonStyle,
+    left: "10px",
+  };
+
+  const nextButtonStyle = {
+    ...buttonStyle,
+    right: "10px",
+  };
+
+  // Overlay style (fixed to fill the screen, flex for centering)
+  const overlayStyle = {
+    position: "fixed",
+    top: 0,
+    left: 0,
+    width: "100vw",
+    height: "100vh",
+    background: "rgba(0,0,0,0.8)",
+    display: "flex",
+    alignItems: "center",
+    justifyContent: "center",
+    opacity: isOpen ? "1" : "0",
+    visibility: isOpen ? "visible" : "hidden",
+    transition: "opacity 0.3s ease, visibility 0.3s ease",
+    zIndex: 9998,
+  };
+
+  // Close button style
+  const closeButtonWrapperStyle = {
+    position: "absolute",
+    top: "10px",
+    right: "10px",
+  };
+
+  const closeButtonStyle = {
+    border: "none",
+    background: "none",
+    color: "#fff",
+    fontSize: "3em",
+    fontWeight: "500",
+    cursor: "pointer",
+    paddingLeft: "1em",
+    paddingBottom: "1em",
+    zIndex: "10000",
+  };
+
+  // Lightbox content container style
+  const contentStyle = {
+    maxWidth: "90%",
+    maxHeight: "90%",
+    display: "flex",
+    alignItems: "center",
+    justifyContent: "center",
+    position: "relative",
+    // fade in/out
+    opacity: isOpen ? "1" : "0",
+    visibility: isOpen ? "visible" : "hidden",
+    transition: "opacity 0.3s ease, visibility 0.3s ease",
+    zIndex: 9999,
+  };
+
+  return html`
+    <div className="lightbox-carousel-container">
+      <!-- Thumbnails -->
+      <div
+        className="carousel-thumbs"
+        style=${{
+          display: "grid",
+          gridTemplateColumns: "auto auto auto auto",
+        }}
+      >
+        ${slides.map((slide, index) => {
+          return html`
+            <div
+              key=${index}
+              className="carousel-thumb"
+              onClick=${() => openLightbox(index)}
+              style=${{
+                background: "black",
+                color: "white",
+                padding: "4em 0",
+                border: "0",
+                margin: "5px",
+                cursor: "pointer",
+                textAlign: "center",
+              }}
+            >
+              <div>${slide.label}</div>
+              <div>
+                <i
+                  class=${ApplicationIcons.play}
+                  style=${{ fontSize: "4em" }}
+                ></i>
+              </div>
+            </div>
+          `;
+        })}
+      </div>
+
+      <!-- Lightbox Overlay -->
+      ${showOverlay &&
+      html`
+        <div className="lightbox-overlay" style=${overlayStyle}>
+          <div style=${closeButtonWrapperStyle}>
+            <button style=${closeButtonStyle} onClick=${closeLightbox}>
+              <i class=${ApplicationIcons.close}></i>
+            </button>
+          </div>
+
+          ${slides.length > 1
+            ? html` <button style=${prevButtonStyle} onClick=${showPrev}>
+                <i class=${ApplicationIcons.previous}></i>
+              </button>`
+            : ""}
+          ${slides.length > 1
+            ? html` <button style=${nextButtonStyle} onClick=${showNext}>
+                <i class=${ApplicationIcons.next}></i>
+              </button>`
+            : ""}
+
+          <div
+            key=${`carousel-slide-${currentIndex}`}
+            className="lightbox-content"
+            style=${contentStyle}
+          >
+            ${slides[currentIndex].render()}
+          </div>
+        </div>
+      `}
+    </div>
+  `;
+};

inspect_ai/_view/www/src/components/Tools.mjs

@@ -51,7 +51,7 @@ export const resolveToolInput = (fn, toolArgs) => {
  * @param {string | undefined } props.input - The main input for this call
  * @param {string | undefined } props.inputType - The input type for this call
  * @param {import("../types/log").ToolCallContent} props.view - The tool call view
- * @param {string | number | boolean | (import("../types/log").ContentText | import("../types/log").ContentImage)[]} props.output - The tool output
+ * @param {string | number | boolean | import("../types/log").ContentText | import("../types/log").ContentImage | (import("../types/log").ContentText | import("../types/log").ContentImage)[]} props.output - The tool output
  * @param { "compact" | undefined } props.mode - The display mode for this call
  * @returns {import("preact").JSX.Element} The SampleTranscript component.
  */
@@ -63,6 +63,14 @@ export const ToolCallView = ({
   output,
   mode,
 }) => {
+  // don't collapse if output includes an image
+  function isContentImage(value) {
+    return value && typeof value === "object" && value.type === "image";
+  }
+  const collapse = Array.isArray(output)
+    ? output.every((item) => !isContentImage(item))
+    : !isContentImage(output);
+
   return html`<div>
     ${mode !== "compact" && (!view || view.title)
       ? html`<${ToolTitle} title=${view?.title || functionCall} />`
@@ -77,7 +85,7 @@ export const ToolCallView = ({
         />
         ${output
           ? html`
-              <${ExpandablePanel} collapse=${true} border=${true} lines=${15}>
+              <${ExpandablePanel} collapse=${collapse} border=${true} lines=${15}>
               <${MessageContent} contents=${normalizeContent(output)} />
               </${ExpandablePanel}>`
           : ""}
@@ -107,7 +115,7 @@ const ToolTitle = ({ title }) => {
 /**
  * Renders the ToolCallView component.
  *
- * @param {string | number | boolean | (import("../types/log").ContentText | import("../types/log").ContentImage)[]} output - The tool output
+ * @param {string | number | boolean | import("../types/log").ContentImage | import("../types/log").ContentText | (import("../types/log").ContentText | import("../types/log").ContentImage)[]} output - The tool output
  * @returns {(import("../Types.mjs").ContentTool | import("../types/log").ContentText | import("../types/log").ContentImage)[]} The SampleTranscript component.
  */
 const normalizeContent = (output) => {

inspect_ai/_view/www/src/samples/transcript/ModelEventView.mjs

@@ -60,10 +60,11 @@ export const ModelEventView = ({ id, event, style }) => {
   };
 
   // For any user messages which immediately preceded this model call, including a
-  // panel and display those user messages
+  // panel and display those user messages (exclude tool_call messages as they
+  // are already shown in the tool call above)
   const userMessages = [];
   for (const msg of event.input.slice().reverse()) {
-    if (msg.role === "user") {
+    if (msg.role === "user" && !msg.tool_call_id) {
       userMessages.push(msg);
     } else {
       break;

inspect_ai/_view/www/src/samples/transcript/TranscriptView.mjs

@@ -155,6 +155,7 @@ export const RenderedEventNode = ({ id, node, style }) => {
         id=${id}
         event=${node.event}
         style=${style}
+        isStore=${true}
       />`;
 
     case "subtask":

inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.mjs

@@ -2,6 +2,7 @@
 import { html } from "htm/preact";
 import { ChatView } from "../../../components/ChatView.mjs";
 import { FontSize, TextStyle } from "../../../appearance/Fonts.mjs";
+import { HumanBaselineView } from "../../../components/HumanBaselineView.mjs";
 
 /**
  * @typedef {Object} Signature
@@ -62,6 +63,58 @@ const add_tools = {
   },
 };
 
+const humanAgentKey = (key) => {
+  return `HumanAgentState:${key}`;
+};
+const human_baseline_session = {
+  type: "human_baseline_session",
+  signature: {
+    add: ["HumanAgentState:logs"],
+    replace: [],
+    remove: [],
+  },
+  render: (changes, resolvedState) => {
+    // Read the session values
+    const started = resolvedState[humanAgentKey("started_running")];
+    const runtime = resolvedState[humanAgentKey("accumulated_time")];
+    const answer = resolvedState[humanAgentKey("answer")];
+    const completed = !!answer;
+    const running = resolvedState[humanAgentKey("running_state")];
+    const rawSessions = resolvedState[humanAgentKey("logs")];
+
+    // Tweak the date value
+    const startedDate = started ? new Date(started * 1000) : undefined;
+
+    // Convert raw sessions into session logs
+    const sessions = {};
+    if (rawSessions) {
+      for (const key of Object.keys(rawSessions)) {
+        const value = rawSessions[key];
+        // this pulls the key apart into
+        // <user>_<timestamp>.<type>
+        const match = key.match(/(.*)_(\d+_\d+)\.(.*)/);
+        if (match) {
+          const user = match[1];
+          const timestamp = match[2];
+          const type = match[3];
+          sessions[timestamp] = sessions[timestamp] || {};
+          sessions[timestamp][type] = value;
+          sessions[timestamp]["user"] = user;
+        }
+      }
+    }
+
+    return html`<${HumanBaselineView}
+      started=${startedDate}
+      running=${running}
+      completed=${completed}
+      answer=${answer}
+      runtime=${runtime}
+      sessionLogs=${Object.values(sessions)}
+    />`;
+  },
+};
+
 const renderTools = (changes, resolvedState) => {
   // Find which tools were added in this change
   const toolIndexes = [];
@@ -136,6 +189,9 @@ export const RenderableChangeTypes = [
   add_tools,
 ];
 
+/** @type {ChangeType[]} */
+export const StoreSpecificRenderableTypes = [human_baseline_session];
+
 /**
  * @typedef {Object} ToolParameters
  * @property {string} type - The type of the parameters object, typically "object".

inspect_ai/_view/www/src/samples/transcript/state/StateEventView.mjs

@@ -2,7 +2,10 @@
 import { html } from "htm/preact";
 
 import { EventPanel } from "../EventPanel.mjs";
-import { RenderableChangeTypes } from "./StateEventRenderers.mjs";
+import {
+  RenderableChangeTypes,
+  StoreSpecificRenderableTypes,
+} from "./StateEventRenderers.mjs";
 import { StateDiffView } from "./StateDiffView.mjs";
 import { formatDateTime } from "../../../utils/Format.mjs";
 
@@ -12,10 +15,11 @@ import { formatDateTime } from "../../../utils/Format.mjs";
  * @param {Object} props - The properties passed to the component.
  * @param { string  } props.id - The id of this event.
  * @param {import("../../../types/log").StateEvent } props.event - The event object to display.
+ * @param { boolean } props.isStore - Whether this event view is rendering a storage (rather than a state)
  * @param { Object } props.style - The style of this event.
  * @returns {import("preact").JSX.Element} The component.
  */
-export const StateEventView = ({ id, event, style }) => {
+export const StateEventView = ({ id, event, isStore, style }) => {
   const summary = summarizeChanges(event.changes);
 
   // Synthesize objects for comparison
@@ -32,7 +36,11 @@ export const StateEventView = ({ id, event, style }) => {
   // This clone is important since the state is used by preact as potential values that are rendered
   // and as a result may be decorated with additional properties, etc..., resulting in DOM elements
   // appearing attached to state.
-  const changePreview = generatePreview(event.changes, structuredClone(after));
+  const changePreview = generatePreview(
+    event.changes,
+    structuredClone(after),
+    isStore,
+  );
   if (changePreview) {
     tabs.unshift(
       html`<div name="Summary" style=${{ margin: "1em 0em", width: "100%" }}>
@@ -55,11 +63,15 @@ export const StateEventView = ({ id, event, style }) => {
  *
  * @param {import("../../../types/log").JsonChange[]} changes - The change object containing the value.
  * @param {Object} resolvedState - The change object containing the value.
+ * @param {boolean} isStore - Is this rendering a store event
  * @returns {import("preact").JSX.Element|Object|string|undefined} - The rendered HTML template if the value is an object with content and source, otherwise the value itself.
  */
-const generatePreview = (changes, resolvedState) => {
+const generatePreview = (changes, resolvedState, isStore) => {
   const results = [];
-  for (const changeType of RenderableChangeTypes) {
+  for (const changeType of [
+    ...RenderableChangeTypes,
+    ...(isStore ? StoreSpecificRenderableTypes : []),
+  ]) {
     // Note that we currently only have renderers that depend upon
     // add, remove, replace, but we should likely add
     // move, copy, test

inspect_ai/_view/www/src/types/asciicinema-player.d.ts

@@ -0,0 +1,26 @@
+declare module "asciinema-player" {
+  export const create: (
+    src: string | Object,
+    el: HTMLElement,
+    opts: {
+      cols?: number;
+      rows?: number;
+      autoPlay?: boolean;
+      preload?: boolean;
+      loop?: boolean;
+      theme?: string;
+      startAt?: number | string;
+      speed?: number;
+      idleTimeLimit?: number;
+      poster?: string;
+      fit?: string;
+      controls?: boolean;
+      markers?: Array<number> | Array<[number, string]>;
+      pauseOnMarkers?: boolean;
+      terminalFontSize?: string;
+      terminalFontFamily?: string;
+      terminalLineHeight?: string;
+      logger?: Object;
+    },
+  ) => any;
+}