inspect-ai 0.3.49__py3-none-any.whl → 0.3.51__py3-none-any.whl

inspect_ai/_view/www/log-schema.json

@@ -2506,6 +2506,18 @@
           ],
           "default": null
         },
+        "time": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Time"
+        },
         "metadata": {
           "anyOf": [
             {
@@ -2537,6 +2549,7 @@
         "model",
         "choices",
         "usage",
+        "time",
         "metadata",
         "error"
       ],

inspect_ai/_view/www/package.json

@@ -36,6 +36,7 @@
     "json5": "^2.2.3",
     "jsondiffpatch": "^0.6.0",
     "markdown-it": "^14.1.0",
+    "murmurhash": "^2.0.1",
     "postcss-url": "^10.1.3",
     "preact": "^10.24.3",
     "prismjs": "^1.29.0"

inspect_ai/_view/www/src/components/MessageBand.mjs

@@ -8,7 +8,7 @@ export const MessageBand = ({ message, hidden, setHidden, type }) => {
   const bgColor =
     type === "info" ? "var(--bs-light)" : "var(--bs-" + type + "-bg-subtle)";
   const color =
-    "var(--bs-" + type === "info" ? "secondary" : undefined + "-text-emphasis)";
+    "var(--bs-" + (type === "info" ? "secondary" : type) + "-text-emphasis)";
 
   return html`
     <div

inspect_ai/_view/www/src/components/Tools.mjs

@@ -1,6 +1,8 @@
 // @ts-check
 /// <reference path="../types/prism.d.ts" />
 import Prism from "prismjs";
+import murmurhash from "murmurhash";
+
 import "prismjs/components/prism-python";
 import "prismjs/components/prism-bash";
 import "prismjs/components/prism-json";
@@ -105,11 +107,11 @@ export const ToolCallView = ({
  * @param {string} props.type - The function call
  * @param {string | undefined } props.contents - The main input for this call
  * @param {Record<string, string>} [props.style] - The style
- * @param {import("../types/log").ToolCallContent} props.view - The tool call view
+ * @param {import("../types/log").ToolCallContent} [props.view] - The tool call view
  * @returns {import("preact").JSX.Element | string} The SampleTranscript component.
  */
 export const ToolInput = ({ type, contents, view, style }) => {
-  if (!contents) {
+  if (!contents && !view?.content) {
     return "";
   }
 
@@ -133,7 +135,7 @@ export const ToolInput = ({ type, contents, view, style }) => {
           }
         }
       }
-    }, [toolInputRef.current]);
+    }, [contents, view, style]);
     return html`<${MarkdownDiv}
       markdown=${view.content}
       ref=${toolInputRef}
@@ -144,14 +146,15 @@ export const ToolInput = ({ type, contents, view, style }) => {
     useEffect(() => {
       const tokens = Prism.languages[type];
       if (toolInputRef.current && tokens) {
-        let resolvedContents = contents;
-        if (typeof contents === "object" || Array.isArray(contents)) {
-          resolvedContents = JSON.stringify(contents);
-        }
-        const html = Prism.highlight(resolvedContents, tokens, type);
-        toolInputRef.current.innerHTML = html;
+        Prism.highlightElement(toolInputRef.current);
       }
-    }, [toolInputRef.current, contents, type, view]);
+    }, [contents, type, view]);
+
+    contents =
+      typeof contents === "object" || Array.isArray(contents)
+        ? JSON.stringify(contents)
+        : contents;
+    const key = murmurhash.v3(contents);
 
     return html`<pre
       class="tool-output"
@@ -162,9 +165,9 @@ export const ToolInput = ({ type, contents, view, style }) => {
         ...style,
       }}
     >
-        <code ref=${toolInputRef} class="sourceCode${type
-      ? ` language-${type}`
-      : ""}" style=${{
+        <code ref=${toolInputRef} 
+          key=${key}
+          class="sourceCode${type ? ` language-${type}` : ""}" style=${{
       overflowWrap: "anywhere",
       whiteSpace: "pre-wrap",
     }}>

inspect_ai/_view/www/src/samples/SampleDisplay.mjs

@@ -150,7 +150,6 @@ export const SampleDisplay = ({
           sample=${sample}
           sampleDescriptor=${sampleDescriptor}
           scorer=${Object.keys(sample.scores)[0]}
-          style=${{ paddingLeft: "0.8em", marginTop: "0.4em" }}
         />
       </${TabPanel}>`);
   } else {
@@ -164,7 +163,6 @@ export const SampleDisplay = ({
             sample=${sample}
             sampleDescriptor=${sampleDescriptor}
             scorer=${scorer}
-            style=${{ paddingLeft: "0.8em", marginTop: "0.4em" }}
           />
         </${TabPanel}>`);
     }
@@ -180,7 +178,7 @@ export const SampleDisplay = ({
           title="Metadata" 
           onSelected=${onSelectedTab} 
           selected=${selectedTab === kSampleMetdataTabId}>
-         <div style=${{ display: "flex", flexWrap: "wrap", alignItems: "flex-start", gap: "1em", paddingLeft: "0.8em", marginTop: "1em" }}> 
+         <div style=${{ display: "flex", flexWrap: "wrap", alignItems: "flex-start", gap: "1em", paddingLeft: "0", marginTop: "0.5em" }}> 
           ${sampleMetadatas}
         </div>
       </${TabPanel}>`,

inspect_ai/_view/www/src/samples/SampleScoreView.mjs

@@ -3,7 +3,8 @@ import { arrayToString, inputString } from "../utils/Format.mjs";
 import { MarkdownDiv } from "../components/MarkdownDiv.mjs";
 import { SampleScores } from "./SampleScores.mjs";
 import { FontSize, TextStyle } from "../appearance/Fonts.mjs";
-import { MetaDataView } from "../components/MetaDataView.mjs";
+import { MetaDataGrid } from "../components/MetaDataGrid.mjs";
+import { Card, CardHeader, CardBody } from "../components/Card.mjs";
 
 const labelStyle = {
   paddingRight: "2em",
@@ -22,6 +23,7 @@ export const SampleScoreView = ({
   if (!sampleDescriptor) {
     return "";
   }
+
   const scoreInput = inputString(sample.input);
   if (sample.choices && sample.choices.length > 0) {
     scoreInput.push("");
@@ -35,17 +37,21 @@ export const SampleScoreView = ({
   const scorerDescriptor = sampleDescriptor.scorer(sample, scorer);
   const explanation = scorerDescriptor.explanation() || "(No Explanation)";
   const answer = scorerDescriptor.answer();
+  const metadata = scorerDescriptor.metadata();
 
   return html`
-    <div
-      class="container-fluid"
-      style=${{
-        paddingTop: "1em",
-        paddingLeft: "0",
-        fontSize: FontSize.base,
-        ...style,
-      }}
-    >
+  <div
+    class="container-fluid"
+    style=${{
+      marginTop: "0.5em",
+      paddingLeft: "0",
+      fontSize: FontSize.base,
+      ...style,
+    }}
+  >
+    <${Card}>
+    <${CardHeader} label="Score"/>
+    <${CardBody}>
       <div>
         <div style=${{ ...labelStyle }}>Input</div>
         <div>
@@ -58,7 +64,7 @@ export const SampleScoreView = ({
 
       <table
         class="table"
-        style=${{ width: "100%", marginBottom: "0", marginTop: "1em" }}
+        style=${{ width: "100%", marginBottom: "1em" }}
       >
         <thead style=${{ borderBottomColor: "#00000000" }}>
           <tr>
@@ -114,73 +120,42 @@ export const SampleScoreView = ({
           </tr>
         </tbody>
       </table>
+    </${CardBody}>
+    </${Card}>
+
+    ${
+      explanation && explanation !== answer
+        ? html` 
+    <${Card}>
+      <${CardHeader} label="Explanation"/>
+      <${CardBody}>
+        <${MarkdownDiv}
+            markdown=${arrayToString(explanation)}
+            style=${{ paddingLeft: "0" }}
+            class="no-last-para-padding"
+          />
+
+      </${CardBody}>
+    </${Card}>`
+        : ""
+    }
 
-      ${explanation && explanation !== answer
-        ? html` <table
-            class="table"
-            style=${{ width: "100%", marginBottom: "0" }}
-          >
-            <thead>
-              <tr>
-                <th
-                  style=${{
-                    paddingBottom: "0",
-                    paddingLeft: "0",
-                    ...labelStyle,
-                    fontWeight: "400",
-                  }}
-                >
-                  Explanation
-                </th>
-              </tr>
-            </thead>
-            <tbody>
-              <tr>
-                <td style=${{ paddingLeft: "0" }}>
-                  <${MarkdownDiv}
-                    markdown=${arrayToString(explanation)}
-                    style=${{ paddingLeft: "0" }}
-                    class="no-last-para-padding"
-                  />
-                </td>
-              </tr>
-            </tbody>
-          </table>`
-        : ""}
-      ${sample?.score?.metadata &&
-      Object.keys(sample?.score?.metadata).length > 0
-        ? html` <table
-            class="table"
-            style=${{ width: "100%", marginBottom: "0" }}
-          >
-            <thead>
-              <tr>
-                <th
-                  style=${{
-                    paddingBottom: "0",
-                    paddingLeft: "0",
-                    ...labelStyle,
-                    fontWeight: "400",
-                  }}
-                >
-                  Metadata
-                </th>
-              </tr>
-            </thead>
-            <tbody>
-              <tr>
-                <td style=${{ paddingLeft: "0" }}>
-                  <${MetaDataView}
-                    id="task-sample-score-metadata"
-                    classes="tab-pane"
-                    entries="${sample?.score?.metadata}"
-                    style=${{ marginTop: "1em" }}
-                  />
-                </td>
-              </tr>
-            </tbody>
-          </table>`
-        : ""}
+    ${
+      metadata && Object.keys(metadata).length > 0
+        ? html`
+    <${Card}>
+      <${CardHeader} label="Metadata"/>
+      <${CardBody}>
+        <${MetaDataGrid}
+          id="task-sample-score-metadata"
+          classes="tab-pane"
+          entries="${metadata}"
+          style=${{ marginTop: "0" }}
+        />
+      </${CardBody}>
+    </${Card}>`
+        : ""
+    }
     </div>
   `;
 };

inspect_ai/_view/www/src/samples/SamplesDescriptor.mjs

@@ -158,6 +158,23 @@ export const createsSamplesDescriptor = (
     }
     return undefined;
   };
+
+  // Retrieve the metadata for a sample
+  /**
+   * @param {import("../api/Types.mjs").SampleSummary} sample - the currently selected score
+   * @param {string} scorer - the scorer name
+   * @returns {Object} The explanation
+   */
+  const scoreMetadata = (sample, scorer) => {
+    if (sample) {
+      const sampleScore = score(sample, scorer);
+      if (sampleScore && sampleScore.metadata) {
+        return sampleScore.metadata;
+      }
+    }
+    return undefined;
+  };
+
   const uniqScoreValues = [
     ...new Set(
       samples
@@ -280,6 +297,9 @@ export const createsSamplesDescriptor = (
 
   const scorerDescriptor = (sample, scorer) => {
     return {
+      metadata: () => {
+        return scoreMetadata(sample, scorer);
+      },
       explanation: () => {
         return scoreExplanation(sample, scorer);
       },
@@ -300,24 +320,18 @@ export const createsSamplesDescriptor = (
         });
         const sampleScorer = sample.scores[scorer];
         const scoreVal = sampleScorer.value;
+
         if (typeof scoreVal === "object") {
           const names = Object.keys(scoreVal);
+
+          // See if this is a dictionary of score names
+          // if any of the score names match, treat it
+          // as a scorer dictionary
           if (
             names.find((name) => {
-              return !scoreNames.includes(name);
+              return scoreNames.includes(name);
             })
           ) {
-            // Since this dictionary contains keys which are not scores
-            // we just treat it like an opaque dictionary
-            return [
-              {
-                name: scorer,
-                rendered: () => {
-                  return scoreDescriptor.render(scoreVal);
-                },
-              },
-            ];
-          } else {
             // Since this dictionary contains keys which are  scores
             // we actually render the individual scores
             const scores = names.map((name) => {
@@ -329,6 +343,17 @@ export const createsSamplesDescriptor = (
               };
             });
             return scores;
+          } else {
+            // Since this dictionary contains keys which are not scores
+            // we just treat it like an opaque dictionary
+            return [
+              {
+                name: scorer,
+                rendered: () => {
+                  return scoreDescriptor.render(scoreVal);
+                },
+              },
+            ];
           }
         } else {
           return [
@@ -389,7 +414,7 @@ const scoreCategorizers = [
      */
     describe: (values) => {
       if (
-        (values.length === 1 || values.length === 2) &&
+        values.length === 2 &&
         values.every((val) => {
           return val === 1 || val === 0;
         })

inspect_ai/_view/www/src/samples/transcript/ModelEventView.mjs

@@ -14,7 +14,11 @@ import { ApplicationIcons } from "../../appearance/Icons.mjs";
 import { MetaDataGrid } from "../../components/MetaDataGrid.mjs";
 import { FontSize, TextStyle } from "../../appearance/Fonts.mjs";
 import { ModelUsagePanel } from "../../usage/UsageCard.mjs";
-import { formatDateTime, formatNumber } from "../../utils/Format.mjs";
+import {
+  formatDateTime,
+  formatNumber,
+  formatPrettyDecimal,
+} from "../../utils/Format.mjs";
 
 /**
  * Renders the StateEventView component.
@@ -28,7 +32,16 @@ import { formatDateTime, formatNumber } from "../../utils/Format.mjs";
  */
 export const ModelEventView = ({ id, event, style }) => {
   const totalUsage = event.output.usage?.total_tokens;
-  const subtitle = totalUsage ? `(${formatNumber(totalUsage)} tokens)` : "";
+  const callTime = event.output.time;
+
+  const subItems = [];
+  if (totalUsage) {
+    subItems.push(`${formatNumber(totalUsage)} tokens`);
+  }
+  if (callTime) {
+    subItems.push(`${formatPrettyDecimal(callTime)} sec`);
+  }
+  const subtitle = subItems.length > 0 ? `(${subItems.join(", ")})` : "";
 
   // Note: despite the type system saying otherwise, this has appeared empircally
   // to sometimes be undefined

inspect_ai/_view/www/src/samples/transcript/state/StateEventRenderers.mjs

@@ -154,7 +154,7 @@ export const RenderableChangeTypes = [
  * @typedef {Object} ToolDefinition
  * @property {string} name - The name of the tool (e.g., "python").
  * @property {string} description - A brief description of what the tool does.
- * @property {ToolParameters} parameters - An object describing the parameters that the tool accepts.
+ * @property {ToolParameters} [parameters] - An object describing the parameters that the tool accepts.
  */
 
 /**
@@ -168,7 +168,9 @@ export const RenderableChangeTypes = [
 export const Tools = ({ toolDefinitions }) => {
   return toolDefinitions.map((toolDefinition) => {
     const toolName = toolDefinition.name;
-    const toolArgs = Object.keys(toolDefinition.parameters.properties);
+    const toolArgs = toolDefinition.parameters?.properties
+      ? Object.keys(toolDefinition.parameters.properties)
+      : [];
     return html`<${Tool} toolName=${toolName} toolArgs=${toolArgs} />`;
   });
 };

inspect_ai/_view/www/src/types/log.d.ts

@@ -173,6 +173,7 @@ export type Logprob2 = number;
 export type Bytes1 = number[] | null;
 export type Content5 = Logprob[];
 export type Choices1 = ChatCompletionChoice[];
+export type Time = number | null;
 export type Metadata4 = {} | null;
 export type Error = string | null;
 export type Scores1 = {
@@ -646,6 +647,7 @@ export interface ModelOutput {
   model: Model1;
   choices: Choices1;
   usage: ModelUsage1 | null;
+  time: Time;
   metadata: Metadata4;
   error: Error;
 }

inspect_ai/_view/www/src/workspace/WorkSpace.mjs

@@ -356,6 +356,8 @@ export const WorkSpace = ({
     logFileName,
     capabilities,
     selectedTab,
+    setHidden,
+    hidden,
   ]);
 
   return html`<${WorkspaceDisplay}

inspect_ai/_view/www/yarn.lock

@@ -1120,6 +1120,11 @@ ms@2.1.2:
   resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
   integrity sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==
 
+murmurhash@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/murmurhash/-/murmurhash-2.0.1.tgz#4097720e08cf978872194ad84ea5be2dec9b610f"
+  integrity sha512-5vQEh3y+DG/lMPM0mCGPDnyV8chYg/g7rl6v3Gd8WMF9S429ox3Xk8qrk174kWhG767KQMqqxLD1WnGd77hiew==
+
 nanoid@^3.3.7:
   version "3.3.7"
   resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8"
@@ -1202,10 +1207,10 @@ postcss@^8.4.40:
     picocolors "^1.0.1"
     source-map-js "^1.2.0"
 
-preact@^10.24.0:
-  version "10.24.2"
-  resolved "https://registry.yarnpkg.com/preact/-/preact-10.24.2.tgz#42179771d3b06e7adb884e3f8127ddd3d99b78f6"
-  integrity sha512-1cSoF0aCC8uaARATfrlz4VCBqE8LwZwRfLgkxJOQwAlQt6ayTmi0D9OF7nXid1POI5SZidFuG9CnlXbDfLqY/Q==
+preact@^10.24.3:
+  version "10.25.1"
+  resolved "https://registry.yarnpkg.com/preact/-/preact-10.25.1.tgz#1c4b84253c42dee874bfbf6a92bdce45e3662665"
+  integrity sha512-frxeZV2vhQSohQwJ7FvlqC40ze89+8friponWUFeVEkaCfhC6Eu4V0iND5C9CXz8JLndV07QRDeXzH1+Anz5Og==
 
 prelude-ls@^1.2.1:
   version "1.2.1"

inspect_ai/approval/__init__.py

@@ -1,7 +1,7 @@
 from ._approval import Approval, ApprovalDecision
 from ._approver import Approver
 from ._auto import auto_approver
-from ._human import human_approver
+from ._human.approver import human_approver
 from ._policy import ApprovalPolicy
 from ._registry import approver
 

inspect_ai/approval/_human/approver.py

@@ -0,0 +1,35 @@
+from inspect_ai.solver._task_state import TaskState
+from inspect_ai.tool._tool_call import ToolCall, ToolCallView
+
+from .._approval import Approval, ApprovalDecision
+from .._approver import Approver
+from .._registry import approver
+from .console import console_approval
+from .panel import panel_approval
+
+
+@approver(name="human")
+def human_approver(
+    choices: list[ApprovalDecision] = ["approve", "reject", "terminate"],
+) -> Approver:
+    """Interactive human approver.
+
+    Returns:
+       Approver: Interactive human approver.
+    """
+
+    async def approve(
+        message: str,
+        call: ToolCall,
+        view: ToolCallView,
+        state: TaskState | None = None,
+    ) -> Approval:
+        # try to use the panel approval (available in fullscreen display)
+        try:
+            return await panel_approval(message, call, view, state, choices)
+
+        # fallback to plain console approval (available in all displays)
+        except NotImplementedError:
+            return console_approval(message, view, choices)
+
+    return approve

inspect_ai/approval/_human/console.py

@@ -0,0 +1,62 @@
+from rich.prompt import Prompt
+
+from inspect_ai._util.transcript import transcript_panel
+from inspect_ai.tool._tool_call import ToolCallView
+from inspect_ai.util._console import input_screen
+
+from .._approval import Approval, ApprovalDecision
+from .util import (
+    HUMAN_APPROVED,
+    HUMAN_ESCALATED,
+    HUMAN_REJECTED,
+    HUMAN_TERMINATED,
+    render_tool_approval,
+)
+
+
+def console_approval(
+    message: str, view: ToolCallView, choices: list[ApprovalDecision]
+) -> Approval:
+    with input_screen(width=None) as console:
+        console.print(
+            transcript_panel(
+                title="Approve Tool", content=render_tool_approval(message, view)
+            )
+        )
+
+        # provide choices
+        prompts: dict[str, str] = {}
+        for choice in choices:
+            prompts[choice[0]] = f"{choice.capitalize()} ({choice[0]})"
+        values = list(prompts.values())
+        prompt = ", ".join(values[:-1])
+        prompt = f"{prompt}, or {values[-1]}"
+
+        def render_approval(approval: Approval) -> Approval:
+            console.print(f"Decision: {approval.decision.capitalize()}")
+            return approval
+
+        while True:
+            decision = Prompt.ask(
+                prompt=prompt,
+                console=console,
+                choices=list(prompts.keys()),
+                default="a",
+            ).lower()
+
+            if decision == "a":
+                return render_approval(
+                    Approval(decision="approve", explanation=HUMAN_APPROVED)
+                )
+            elif decision == "r":
+                return render_approval(
+                    Approval(decision="reject", explanation=HUMAN_REJECTED)
+                )
+            elif decision == "t":
+                return render_approval(
+                    Approval(decision="terminate", explanation=HUMAN_TERMINATED)
+                )
+            elif decision == "e":
+                return render_approval(
+                    Approval(decision="escalate", explanation=HUMAN_ESCALATED)
+                )