inspect-ai 0.3.52__py3-none-any.whl → 0.3.53__py3-none-any.whl

inspect_ai/_eval/task/sandbox.py

@@ -1,7 +1,8 @@
 import asyncio
 import base64
 import contextlib
-from typing import AsyncGenerator, NamedTuple
+from random import random
+from typing import AsyncGenerator, Callable, NamedTuple, cast
 
 from inspect_ai._eval.task.task import Task
 from inspect_ai._eval.task.util import task_run_dir
@@ -9,6 +10,7 @@ from inspect_ai._util.file import file, filesystem
 from inspect_ai._util.registry import registry_unqualified_name
 from inspect_ai._util.url import data_uri_to_base64, is_data_uri
 from inspect_ai.dataset import Sample
+from inspect_ai.util._concurrency import concurrency
 from inspect_ai.util._sandbox.context import (
     cleanup_sandbox_environments_sample,
     init_sandbox_environments_sample,
@@ -18,12 +20,14 @@ from inspect_ai.util._sandbox.environment import (
     SandboxEnvironmentConfigType,
     SandboxEnvironmentSpec,
 )
+from inspect_ai.util._sandbox.registry import registry_find_sandboxenv
 
 
 @contextlib.asynccontextmanager
 async def sandboxenv_context(
     task_name: str,
     sandbox: SandboxEnvironmentSpec | None,
+    max_sandboxes: int | None,
     cleanup: bool,
     sample: Sample,
 ) -> AsyncGenerator[None, None]:
@@ -32,52 +36,77 @@ async def sandboxenv_context(
     if not sandbox:
         raise ValueError("sandboxenv_context called with no sandbox specified")
 
-    # read files from sample
-    files: dict[str, bytes] = {}
-    if sample.files:
-        for path, contents in sample.files.items():
-            files[path] = read_sandboxenv_file(contents)
-
-    # read setup script from sample (add bash shebang if necessary)
-    setup: bytes | None = None
-    if sample.setup:
-        setup = read_sandboxenv_file(sample.setup)
-        setup_str = setup.decode(encoding="utf-8")
-        if not setup_str.strip().startswith("#!"):
-            setup_str = f"#!/usr/bin/env bash\n\n{setup_str}"
-            setup = setup_str.encode(encoding="utf-8")
-
-    interrupted = False
-    environments: dict[str, SandboxEnvironment] | None = None
-    try:
-        # initialize sandbox environment,
-        environments = await init_sandbox_environments_sample(
-            type=sandbox.type,
-            task_name=registry_unqualified_name(task_name),
-            config=sandbox.config,
-            files=files,
-            setup=setup,
-            metadata=sample.metadata if sample.metadata else {},
-        )
-
-        # run sample
-        yield
-
-    except asyncio.CancelledError as ex:
-        interrupted = True
-        raise ex
+    # get sandboxenv_type
+    sandboxenv_type = registry_find_sandboxenv(sandbox.type)
 
-    finally:
-        # cleanup sandbox environment
-        if environments and cleanup:
-            await cleanup_sandbox_environments_sample(
-                type=sandbox.type,
-                task_name=task_name,
+    # see if there is a max_sandboxes in play (passed or from type)
+    if max_sandboxes is None:
+        default_concurrency_fn = cast(
+            Callable[[], int | None], getattr(sandboxenv_type, "default_concurrency")
+        )
+        max_sandboxes = default_concurrency_fn()
+
+    # if we are enforcing max_sandboxes, then when samples are scheduled they may
+    # not get interleaved properly across tasks (because the first task will come
+    # in and grab all of the sandboxes). Therefore, in this case we wait a random
+    # delay so that all tasks/samples have an equal shot at getting scheduled.
+    if max_sandboxes is not None:
+        await asyncio.sleep(random())
+
+    # enforce concurrency if required
+    sandboxes_cm = (
+        concurrency(sandbox.type, max_sandboxes, f"sandboxes/{sandbox.type}")
+        if max_sandboxes is not None
+        else contextlib.nullcontext()
+    )
+
+    async with sandboxes_cm:
+        # read files from sample
+        files: dict[str, bytes] = {}
+        if sample.files:
+            for path, contents in sample.files.items():
+                files[path] = read_sandboxenv_file(contents)
+
+        # read setup script from sample (add bash shebang if necessary)
+        setup: bytes | None = None
+        if sample.setup:
+            setup = read_sandboxenv_file(sample.setup)
+            setup_str = setup.decode(encoding="utf-8")
+            if not setup_str.strip().startswith("#!"):
+                setup_str = f"#!/usr/bin/env bash\n\n{setup_str}"
+                setup = setup_str.encode(encoding="utf-8")
+
+        interrupted = False
+        environments: dict[str, SandboxEnvironment] | None = None
+        try:
+            # initialize sandbox environment,
+            environments = await init_sandbox_environments_sample(
+                sandboxenv_type=sandboxenv_type,
+                task_name=registry_unqualified_name(task_name),
                 config=sandbox.config,
-                environments=environments,
-                interrupted=interrupted,
+                files=files,
+                setup=setup,
+                metadata=sample.metadata if sample.metadata else {},
             )
 
+            # run sample
+            yield
+
+        except asyncio.CancelledError as ex:
+            interrupted = True
+            raise ex
+
+        finally:
+            # cleanup sandbox environment
+            if environments and cleanup:
+                await cleanup_sandbox_environments_sample(
+                    type=sandbox.type,
+                    task_name=task_name,
+                    config=sandbox.config,
+                    environments=environments,
+                    interrupted=interrupted,
+                )
+
 
 def read_sandboxenv_file(contents: str) -> bytes:
     if is_data_uri(contents):

inspect_ai/_eval/task/task.py

@@ -39,6 +39,8 @@ class Task:
 
     Args:
         dataset (Dataset | Sequence[Sample]): Dataset to evaluate
+        setup: (Solver | list[Solver] | None): Setup step (always run
+          even when the main `solver` is replaced).
         solver: (Solver | list[Solver]): Solver or list of solvers.
           Defaults to generate(), a normal call to the model.
         scorer: (Scorer | list[Scorer] | None): Scorer used to evaluate model output.
@@ -68,6 +70,7 @@ class Task:
     def __init__(
         self,
         dataset: Dataset | Sequence[Sample] | None = None,
+        setup: Solver | list[Solver] | None = None,
         solver: Solver | list[Solver] = generate(),
         scorer: Scorer | list[Scorer] | None = None,
         metrics: list[Metric] | dict[str, list[Metric]] | None = None,
@@ -119,6 +122,7 @@ class Task:
         self.dataset: Dataset = (
             dataset if isinstance(dataset, Dataset) else MemoryDataset(list(dataset))
         )
+        self.setup = setup
         self.solver = chain(solver) if isinstance(solver, list) else solver
         self.scorer = (
             scorer

inspect_ai/_eval/task/util.py

@@ -42,6 +42,8 @@ def slice_dataset(
     sample_id: str | int | list[str | int] | None,
 ) -> Dataset:
     def normalise(id: str | int | None) -> str:
+        if isinstance(id, str) and id.isdigit():
+            id = int(id)
         return id if isinstance(id, str) else str(id).zfill(20)
 
     if sample_id is not None:

inspect_ai/_view/www/App.css

@@ -711,6 +711,19 @@ pre[class*="language-"].tool-output,
   background-color: #333333;
 }
 
+pre[class*="language-"].tool-output {
+  border: none !important;
+  box-shadow: none !important;
+  border-radius: var(--bs-border-radius) !important;
+}
+
+.vscode-dark pre.jsonPanel {
+  background: none !important;
+  border: none !important;
+  box-shadow: none !important;
+  border-radius: var(--bs-border-radius) !important;  
+}
+
 
 /* jsondiffpatch */
 

inspect_ai/_view/www/dist/assets/index.css

@@ -14984,6 +14984,19 @@ pre[class*="language-"].tool-output,
   background-color: #333333;
 }
 
+pre[class*="language-"].tool-output {
+  border: none !important;
+  box-shadow: none !important;
+  border-radius: var(--bs-border-radius) !important;
+}
+
+.vscode-dark pre.jsonPanel {
+  background: none !important;
+  border: none !important;
+  box-shadow: none !important;
+  border-radius: var(--bs-border-radius) !important;  
+}
+
 
 /* jsondiffpatch */
 

inspect_ai/_view/www/dist/assets/index.js

@@ -15251,21 +15251,10 @@ const ToolCallView = ({
   output,
   mode
 }) => {
-  const icon = mode === "compact" ? "" : m$1`<i
-          class="bi bi-tools"
-          style=${{
-    marginRight: "0.2rem",
-    opacity: "0.4"
-  }}
-        ></i>`;
-  const codeIndent = mode === "compact" ? "" : "";
   return m$1`<div>
-    ${icon}
-    ${!view || view.title ? m$1`<code style=${{ fontSize: FontSize.small }}
-          >${(view == null ? void 0 : view.title) || functionCall}</code
-        >` : ""}
+    ${mode !== "compact" && (!view || view.title) ? m$1`<${ToolTitle} title=${(view == null ? void 0 : view.title) || functionCall} />` : ""}
     <div>
-      <div style=${{ marginLeft: `${codeIndent}` }}>
+      <div>
         <${ToolInput}
           type=${inputType}
           contents=${input}
@@ -15274,12 +15263,39 @@ const ToolCallView = ({
         />
         ${output ? m$1`
               <${ExpandablePanel} collapse=${true} border=${true} lines=${15}>
-              <${MessageContent} contents=${output} />
+              <${MessageContent} contents=${normalizeContent$1(output)} />
               </${ExpandablePanel}>` : ""}
       </div>
     </div>
   </div>`;
 };
+const ToolTitle = ({ title }) => {
+  return m$1` <i
+      class="bi bi-tools"
+      style=${{
+    marginRight: "0.2rem",
+    opacity: "0.4"
+  }}
+    ></i>
+    <code style=${{ fontSize: FontSize.small }}>${title}</code>`;
+};
+const normalizeContent$1 = (output) => {
+  if (Array.isArray(output)) {
+    return output;
+  } else {
+    return [
+      {
+        type: "tool",
+        content: [
+          {
+            type: "text",
+            text: String(output)
+          }
+        ]
+      }
+    ];
+  }
+};
 const ToolInput = ({ type, contents, view, style }) => {
   if (!contents && !(view == null ? void 0 : view.content)) {
     return "";
@@ -15455,8 +15471,7 @@ const extractInput = (inputKey, args) => {
     args: []
   };
 };
-const MessageContent = (props) => {
-  const { contents } = props;
+const MessageContent = ({ contents }) => {
   if (Array.isArray(contents)) {
     return contents.map((content, index) => {
       if (typeof content === "string") {
@@ -19422,21 +19437,15 @@ const LoggerEventView = ({ id, event, style }) => {
 };
 const kPrismRenderMaxSize = 25e4;
 const JSONPanel = ({ id, json, data, simple, style }) => {
-  const sourceCode = json || JSON.stringify(data, void 0, 2);
   const codeRef = A();
-  if (codeRef.current) {
+  const sourceCode = T(() => {
+    return json || JSON.stringify(data, void 0, 2);
+  }, [json, data]);
+  y(() => {
     if (sourceCode.length < kPrismRenderMaxSize) {
-      codeRef.current.innerHTML = Prism$1.highlight(
-        sourceCode,
-        Prism$1.languages.javascript,
-        "javacript"
-      );
-    } else {
-      const textNode = document.createTextNode(sourceCode);
-      codeRef.current.innerText = "";
-      codeRef.current.appendChild(textNode);
+      Prism$1.highlightElement(codeRef.current);
     }
-  }
+  }, [sourceCode]);
   return m$1`<div>
     <pre
       style=${{
@@ -19446,16 +19455,18 @@ const JSONPanel = ({ id, json, data, simple, style }) => {
     borderRadius: simple ? void 0 : "var(--bs-border-radius)",
     ...style
   }}
+      class="jsonPanel"
     >
     <code 
       id=${id}
       ref=${codeRef}
-      class="sourceCode-json" 
+      class="sourceCode language-javascript" 
       style=${{
     fontSize: FontSize.small,
     whiteSpace: "pre-wrap",
     wordWrap: "anywhere"
   }}>
+      ${sourceCode}
     </code>
     </pre>
   </div>`;
@@ -19569,6 +19580,7 @@ const decisionIcon = (decision) => {
   }
 };
 const ToolEventView = ({ id, event, style, depth }) => {
+  var _a2;
   const { input, functionCall, inputType } = resolveToolInput(
     event.function,
     event.arguments
@@ -19576,10 +19588,10 @@ const ToolEventView = ({ id, event, style, depth }) => {
   const approvalEvent = event.events.find((e2) => {
     return e2.event === "approval";
   });
-  const title = `Tool: ${event.function}`;
+  const title = `Tool: ${((_a2 = event.view) == null ? void 0 : _a2.title) || event.function}`;
   return m$1`
   <${EventPanel} id=${id} title="${title}" subTitle=${formatDateTime(new Date(event.timestamp))} icon=${ApplicationIcons.solvers.use_tools} style=${style}>  
-  <div name="Summary" style=${{ margin: "0.5em 0" }}>
+  <div name="Summary" style=${{ margin: "0.5em 0", width: "100%" }}>
     <${ToolCallView}
       functionCall=${functionCall}
       input=${input}
@@ -20267,7 +20279,10 @@ const SampleSummary = ({ parent_id, sample, style, sampleDescriptor }) => {
       clamp: true
     });
   }
-  const fullAnswer = sample && sampleDescriptor ? sampleDescriptor.selectedScorer(sample).answer() : void 0;
+  const fullAnswer = sample && sampleDescriptor ? (
+    // @ts-ignore
+    sampleDescriptor.selectedScorer(sample).answer()
+  ) : void 0;
   if (fullAnswer) {
     columns.push({
       label: "Answer",
@@ -20293,7 +20308,11 @@ const SampleSummary = ({ parent_id, sample, style, sampleDescriptor }) => {
     value: sample.error ? m$1`<${FlatSampleError}
           message=${sample.error.message}
           style=${{ marginTop: "0.4rem" }}
-        />` : sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render(),
+        />` : (
+      // TODO: Cleanup once the PR lands which makes sample / sample summary share common interface
+      // @ts-ignore
+      sampleDescriptor == null ? void 0 : sampleDescriptor.selectedScore(sample).render()
+    ),
     size: "minmax(2em, auto)",
     center: true
   });
@@ -20601,7 +20620,7 @@ const SampleList = (props) => {
     [selectedIndex]
   );
   const listStyle = { ...style, flex: "1", overflowY: "auto", outline: "none" };
-  const { limit, answer } = gridColumns(sampleDescriptor);
+  const { limit, answer, target } = gridColumns(sampleDescriptor);
   const headerRow = m$1`<div
     style=${{
     display: "grid",
@@ -20616,7 +20635,7 @@ const SampleList = (props) => {
   >
     <div>Id</div>
     <div>Input</div>
-    <div>Target</div>
+    <div>${target !== "0" ? "Target" : ""}</div>
     <div>${answer !== "0" ? "Answer" : ""}</div>
     <div>${limit !== "0" ? "Limit" : ""}</div>
     <div style=${{ justifySelf: "center" }}>Score</div>
@@ -24620,7 +24639,9 @@ const ResultsPanel = ({ results }) => {
       flexWrap: "wrap",
       justifyContent: "end",
       height: "100%",
-      alignItems: "center"
+      alignItems: "center",
+      maxHeight: "15em",
+      overflow: "scroll"
     }}
     >
       ${metrics.map((metric, i) => {
@@ -24638,7 +24659,9 @@ const ResultsPanel = ({ results }) => {
       alignItems: "center",
       marginTop: "0.2rem",
       paddingBottom: "0.4rem",
-      rowGap: "1em"
+      rowGap: "1em",
+      maxHeight: "15em",
+      overflow: "scroll"
     }}
     >
       ${(_b2 = results == null ? void 0 : results.scores) == null ? void 0 : _b2.map((score, index) => {
@@ -26223,8 +26246,12 @@ function App({
           }
         }
       } catch (e2) {
-        console.log(e2);
-        setStatus({ loading: false, error: e2 });
+        if (e2.message === "Load failed" || e2.message === "Failed to fetch") {
+          setStatus({ loading: false });
+        } else {
+          console.log(e2);
+          setStatus({ loading: false, error: e2 });
+        }
       }
       setHeadersLoading(false);
     };
@@ -26452,8 +26479,11 @@ function App({
       }
       new ClipboardJS(".clipboard-button,.copy-button");
       if (pollForLogs) {
-        setInterval(() => {
-          api2.client_events().then(async (events) => {
+        let retryDelay = 1e3;
+        const maxRetryDelay = 6e4;
+        const pollEvents = async () => {
+          try {
+            const events = await api2.client_events();
             if (events.includes("reload")) {
               window.location.reload();
             }
@@ -26462,8 +26492,15 @@ function App({
               setLogs(logs2);
               setSelectedLogIndex(0);
             }
-          });
-        }, 1e3);
+            retryDelay = 1e3;
+          } catch (error2) {
+            console.error("Error fetching client events:", error2);
+            retryDelay = Math.min(retryDelay * 2, maxRetryDelay);
+          } finally {
+            setTimeout(pollEvents, retryDelay);
+          }
+        };
+        pollEvents();
       }
     };
     loadLogsAndState();

inspect_ai/_view/www/src/App.mjs

@@ -480,8 +480,13 @@ export function App({
           }
         }
       } catch (e) {
-        console.log(e);
-        setStatus({ loading: false, error: e });
+        if (e.message === "Load failed" || e.message === "Failed to fetch") {
+          // This will happen if the server disappears (e.g. inspect view is terminated)
+          setStatus({ loading: false });
+        } else {
+          console.log(e);
+          setStatus({ loading: false, error: e });
+        }
       }
 
       setHeadersLoading(false);
@@ -774,18 +779,38 @@ export function App({
       new ClipboardJS(".clipboard-button,.copy-button");
 
       if (pollForLogs) {
-        setInterval(() => {
-          api.client_events().then(async (events) => {
+        let retryDelay = 1000;
+        const maxRetryDelay = 60000;
+
+        const pollEvents = async () => {
+          try {
+            const events = await api.client_events();
+
             if (events.includes("reload")) {
               window.location.reload();
             }
+
             if (events.includes("refresh-evals")) {
               const logs = await load();
               setLogs(logs);
               setSelectedLogIndex(0);
             }
-          });
-        }, 1000);
+
+            // Reset delay after a successful call
+            retryDelay = 1000;
+          } catch (error) {
+            console.error("Error fetching client events:", error);
+
+            // Exponential backoff with capping
+            retryDelay = Math.min(retryDelay * 2, maxRetryDelay);
+          } finally {
+            // Schedule the next poll
+            setTimeout(pollEvents, retryDelay);
+          }
+        };
+
+        // Start polling
+        pollEvents();
       }
     };
 

inspect_ai/_view/www/src/Types.mjs

@@ -32,3 +32,9 @@
 /**
  * @typedef {"none" | "single" | "many"} SampleMode
  */
+
+/**
+ * @typedef {Object} ContentTool
+ * @property {"tool"} type
+ * @property {(import("./types/log").ContentImage | import("./types/log").ContentText)[]} content
+ */

inspect_ai/_view/www/src/components/JsonPanel.mjs

@@ -4,7 +4,7 @@ import Prism from "prismjs";
 import "prismjs/components/prism-json";
 
 import { html } from "htm/preact";
-import { useRef } from "preact/hooks";
+import { useEffect, useMemo, useRef } from "preact/hooks";
 import { FontSize } from "../appearance/Fonts.mjs";
 
 const kPrismRenderMaxSize = 250000;
@@ -22,25 +22,17 @@ const kPrismRenderMaxSize = 250000;
  * @returns {import('preact').JSX.Element} The rendered component.
  */
 export const JSONPanel = ({ id, json, data, simple, style }) => {
-  const sourceCode = json || JSON.stringify(data, undefined, 2);
   const codeRef = useRef();
 
-  if (codeRef.current) {
+  const sourceCode = useMemo(() => {
+    return json || JSON.stringify(data, undefined, 2);
+  }, [json, data]);
+
+  useEffect(() => {
     if (sourceCode.length < kPrismRenderMaxSize) {
-      // @ts-ignore
-      codeRef.current.innerHTML = Prism.highlight(
-        sourceCode,
-        Prism.languages.javascript,
-        "javacript",
-      );
-    } else {
-      const textNode = document.createTextNode(sourceCode);
-      // @ts-ignore
-      codeRef.current.innerText = "";
-      // @ts-ignore
-      codeRef.current.appendChild(textNode);
+      Prism.highlightElement(codeRef.current);
     }
-  }
+  }, [sourceCode]);
 
   return html`<div>
     <pre
@@ -51,16 +43,18 @@ export const JSONPanel = ({ id, json, data, simple, style }) => {
         borderRadius: simple ? undefined : "var(--bs-border-radius)",
         ...style,
       }}
+      class="jsonPanel"
     >
     <code 
       id=${id}
       ref=${codeRef}
-      class="sourceCode-json" 
+      class="sourceCode language-javascript" 
       style=${{
       fontSize: FontSize.small,
       whiteSpace: "pre-wrap",
       wordWrap: "anywhere",
     }}>
+      ${sourceCode}
     </code>
     </pre>
   </div>`;

inspect_ai/_view/www/src/components/MessageContent.mjs

@@ -2,8 +2,15 @@ import { html } from "htm/preact";
 import { MarkdownDiv } from "./MarkdownDiv.mjs";
 import { ToolOutput } from "./Tools.mjs";
 
-export const MessageContent = (props) => {
-  const { contents } = props;
+/**
+ * Renders message content based on its type.
+ * Supports rendering strings, images, and tools using specific renderers.
+ *
+ * @param {Object} props - The props object.
+ * @param {string|string[]| (import("../types/log").ContentText | import("../types/log").ContentImage | import("../Types.mjs").ContentTool)[]} props.contents - The content or array of contents to render.
+ * @returns {import("preact").JSX.Element | import("preact").JSX.Element[]} The component.
+ */
+export const MessageContent = ({ contents }) => {
   if (Array.isArray(contents)) {
     return contents.map((content, index) => {
       if (typeof content === "string") {