inspect-ai 0.3.91__py3-none-any.whl → 0.3.93__py3-none-any.whl

inspect_ai/_view/www/src/app/plan/PlanDetailView.tsx

@@ -1,14 +1,10 @@
 import { EvalPlan, EvalScore, EvalSpec, Params2 } from "../../@types/log";
-import { toTitleCase } from "../../utils/format";
-import { ghCommitUrl } from "../../utils/git";
-import { MetaDataView } from "../content/MetaDataView";
 import { DatasetDetailView } from "./DatasetDetailView";
 import { ScorerDetailView } from "./ScorerDetailView";
 import { SolversDetailView } from "./SolverDetailView";
 
 import clsx from "clsx";
 import { FC, ReactNode } from "react";
-import { kModelNone } from "../../constants";
 import styles from "./PlanDetailView.module.css";
 
 interface PlanDetailViewProps {
@@ -26,71 +22,7 @@ export const PlanDetailView: FC<PlanDetailViewProps> = ({
     return null;
   }
 
-  // Add configuration
-  const config: Record<string, unknown> = {};
-  Object.entries(evaluation?.config || {}).forEach((entry) => {
-    const key = entry[0];
-    const value = entry[1];
-    config[key] = value;
-  });
-
   const steps = plan?.steps;
-  const metadata = evaluation?.metadata;
-  const revision = evaluation?.revision;
-  const packages = evaluation?.packages;
-  const model_args = evaluation?.model_args;
-  const task_args = evaluation?.task_args;
-  const generate_config = plan?.config;
-
-  const taskInformation: Record<string, unknown> = {
-    ["Task ID"]: evaluation?.task_id,
-    ["Run ID"]: evaluation?.run_id,
-  };
-  if (revision) {
-    taskInformation[
-      `${revision.type ? `${toTitleCase(revision.type)} ` : ""}Revision`
-    ] = {
-      _html: (
-        <a href={ghCommitUrl(revision.origin, revision.commit)}>
-          {revision.commit}
-        </a>
-      ),
-    };
-  }
-  if (packages) {
-    const names = Object.keys(packages).map((key) => {
-      return `${key} ${packages[key]}`;
-    });
-
-    if (names.length === 1) {
-      taskInformation["Inspect"] = names[0];
-    } else {
-      taskInformation["Inspect"] = names;
-    }
-  }
-  if (evaluation.tags) {
-    taskInformation["Tags"] = evaluation.tags.join(", ");
-  }
-
-  if (evaluation?.model && evaluation.model !== kModelNone) {
-    config["model"] = evaluation.model;
-  }
-
-  if (evaluation?.model_base_url) {
-    config["model_base_url"] = evaluation.model_base_url;
-  }
-
-  if (evaluation?.sandbox) {
-    if (Array.isArray(evaluation?.sandbox)) {
-      config["sandbox"] = evaluation.sandbox[0];
-      if (evaluation.sandbox[1]) {
-        config["sandbox_config"] = evaluation.sandbox[1];
-      }
-    } else {
-      config["sandbox"] = evaluation?.sandbox.type;
-      config["sandbox_config"] = evaluation?.sandbox.config;
-    }
-  }
 
   const taskColumns: {
     title: string;
@@ -148,117 +80,12 @@ export const PlanDetailView: FC<PlanDetailViewProps> = ({
     }
   }
 
-  // Compute the column style for the remaining (either 1 or 2 columns wide)
-  const metadataColumns: {
-    title: string;
-    className: string;
-    contents: ReactNode;
-  }[] = [];
-  const cols = colCount(
-    metadataColumns,
-    task_args,
-    model_args,
-    config,
-    metadata,
-  );
-
-  metadataColumns.push({
-    title: "Task Information",
-    className: cols === 1 ? styles.oneCol : styles.twoCol,
-    contents: (
-      <MetaDataView
-        key={`plan-md-task`}
-        className={"text-size-small"}
-        entries={taskInformation}
-        tableOptions="sm"
-      />
-    ),
-  });
-
-  if (task_args && Object.keys(task_args).length > 0) {
-    metadataColumns.push({
-      title: "Task Args",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-task-args`}
-          className={"text-size-small"}
-          entries={task_args as Record<string, unknown>}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-  if (model_args && Object.keys(model_args).length > 0) {
-    metadataColumns.push({
-      title: "Model Args",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-model-args`}
-          className={"text-size-small"}
-          entries={model_args as Record<string, unknown>}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-
-  if (config && Object.keys(config).length > 0) {
-    metadataColumns.push({
-      title: "Configuration",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-config`}
-          className={"text-size-small"}
-          entries={config}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-
-  if (generate_config && Object.keys(generate_config).length > 0) {
-    const generate_record: Record<string, unknown> = Object.fromEntries(
-      Object.entries(generate_config),
-    );
-
-    metadataColumns.push({
-      title: "Generate Config",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-generate-config`}
-          className={"text-size-small"}
-          entries={generate_record}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-
-  if (metadata && Object.keys(metadata).length > 0) {
-    metadataColumns.push({
-      title: "Metadata",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-metadata`}
-          className={"text-size-small"}
-          entries={metadata}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-
   return (
     <div className={styles.container}>
       <div
         className={styles.grid}
         style={{
-          gridTemplateColumns: `repeat(${taskColumns.length}, auto)`,
+          gridTemplateColumns: `repeat(${taskColumns.length}, fit-content(50%))`,
         }}
       >
         {taskColumns.map((col) => {
@@ -273,34 +100,10 @@ export const PlanDetailView: FC<PlanDetailViewProps> = ({
           );
         })}
       </div>
-
-      <div className={clsx(styles.row)}>
-        {metadataColumns.map((col) => {
-          return (
-            <PlanColumn
-              title={col.title}
-              className={col.className}
-              key={`plan-col-${col.title}`}
-            >
-              {col.contents}
-            </PlanColumn>
-          );
-        })}
-      </div>
     </div>
   );
 };
 
-const colCount = (...other: unknown[]) => {
-  let count = 0;
-  for (const o in other) {
-    if (o && Object.keys(o).length > 0) {
-      count++;
-    }
-  }
-  return count;
-};
-
 interface PlanColumnProps {
   title: string;
   className: string | string[];

inspect_ai/_view/www/src/app/samples/descriptor/score/NumericScoreDescriptor.tsx

@@ -1,6 +1,7 @@
 import { Value2 } from "../../../../@types/log";
 import { kScoreTypeNumeric } from "../../../../constants";
 import { formatDecimalNoTrailingZeroes } from "../../../../utils/format";
+import { compareWithNan } from "../../../../utils/numeric";
 import { ScoreDescriptor } from "../types";
 
 export const numericScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
@@ -14,7 +15,7 @@ export const numericScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
     max: Math.max(...onlyNumeric),
     compare: (a, b) => {
       if (typeof a.value === "number" && typeof b.value === "number") {
-        return a.value - b.value;
+        return compareWithNan(a.value, b.value);
       } else {
         console.warn("Comparing non-numerics using a numeric score descriptor");
         return 0;

inspect_ai/_view/www/src/app/usage/ModelUsagePanel.tsx

@@ -6,6 +6,7 @@ import styles from "./ModelUsagePanel.module.css";
 
 interface ModelUsageProps {
   usage: ModelUsage1;
+  className?: string | string[];
 }
 
 interface ModelUsageRow {
@@ -19,7 +20,7 @@ interface ModelUsageRow {
 /**
  * Renders the ModelUsagePanel component.
  */
-export const ModelUsagePanel: FC<ModelUsageProps> = ({ usage }) => {
+export const ModelUsagePanel: FC<ModelUsageProps> = ({ usage, className }) => {
   if (!usage) {
     return null;
   }
@@ -84,7 +85,7 @@ export const ModelUsagePanel: FC<ModelUsageProps> = ({ usage }) => {
   });
 
   return (
-    <div className={clsx("text-size-small", styles.wrapper)}>
+    <div className={clsx("text-size-small", styles.wrapper, className)}>
       {rows.map((row, idx) => {
         if (row.label === "---") {
           return (

inspect_ai/_view/www/src/app/usage/TokenTable.module.css

@@ -1,6 +1,5 @@
 .table {
   width: 100%;
-  margin-top: 0.7rem;
 }
 
 .tableTokens {
@@ -15,3 +14,7 @@
 .model {
   padding-right: 1em;
 }
+
+.cellContents {
+  padding-bottom: 1em;
+}

inspect_ai/_view/www/src/app/usage/TokenTable.tsx

@@ -79,10 +79,10 @@ export const TokenRow: FC<TokenRowProps> = ({ model, usage }) => {
   return (
     <tr>
       <td>
-        <div className={styles.model}>{model}</div>
+        <div className={clsx(styles.model, styles.cellContents)}>{model}</div>
       </td>
       <td>
-        <ModelUsagePanel usage={usage} />
+        <ModelUsagePanel usage={usage} className={clsx(styles.cellContents)} />
       </td>
     </tr>
   );

inspect_ai/_view/www/src/app/usage/UsageCard.module.css

@@ -3,13 +3,18 @@
   padding-bottom: 1em;
   margin-left: 0.5em;
   display: flex;
+  flex-wrap: wrap;
+  gap: 1em;
 }
 
 .col1 {
-  flex: 1 1 40%;
-  margin-right: 1em;
+  flex: 0 1 auto;
+  min-width: 200px;
+  width: fit-content;
 }
 
 .col2 {
-  flex: 1 1 60%;
+  flex: 1 1 auto;
+  min-width: 300px;
+  width: fit-content;
 }

inspect_ai/_view/www/src/app/usage/UsageCard.tsx

@@ -1,10 +1,5 @@
-import clsx from "clsx";
 import { EvalStats } from "../../@types/log";
-import { FontSize } from "../../app/appearance/fonts";
-import { ApplicationIcons } from "../../app/appearance/icons";
-import { MetaDataView } from "../../app/content/MetaDataView";
 import { Card, CardBody, CardHeader } from "../../components/Card";
-import { formatDuration } from "../../utils/format";
 import { ModelTokenTable } from "./ModelTokenTable";
 
 import { FC } from "react";
@@ -24,40 +19,11 @@ export const UsageCard: FC<UsageCardProps> = ({ stats }) => {
     return null;
   }
 
-  const totalDuration = formatDuration(
-    new Date(stats.started_at),
-    new Date(stats.completed_at),
-  );
-  const usageMetadataStyle = {
-    fontSize: FontSize.smaller,
-  };
-
   return (
     <Card>
-      <CardHeader icon={ApplicationIcons.usage} label="Usage" />
+      <CardHeader label="Usage" />
       <CardBody id={kUsageCardBodyId}>
         <div className={styles.wrapper}>
-          <div className={styles.col1}>
-            <div
-              className={clsx(
-                "text-size-smaller",
-                "text-style-label",
-                "text-style-secondary",
-              )}
-            >
-              Duration
-            </div>
-            <MetaDataView
-              entries={{
-                ["Start"]: new Date(stats.started_at).toLocaleString(),
-                ["End"]: new Date(stats.completed_at).toLocaleString(),
-                ["Duration"]: totalDuration,
-              }}
-              tableOptions="borderless,sm"
-              style={usageMetadataStyle}
-            />
-          </div>
-
           <div className={styles.col2}>
             <ModelTokenTable model_usage={stats.model_usage} />
           </div>

inspect_ai/_view/www/src/components/Card.css

@@ -23,7 +23,6 @@
   background-color: var(--bs-light-bg-subtle);
   border: solid 1px var(--bs-light-border-subtle);
   border-radius: var(--bs-border-radius);
-  margin-bottom: 1.5em;
 }
 
 .card-collaping-header {

inspect_ai/_view/www/src/constants.ts

@@ -5,6 +5,8 @@ export const kModelNone = "none/none";
 export const kLogViewSamplesTabId = "samples";
 export const kLogViewJsonTabId = "json";
 export const kLogViewInfoTabId = "info";
+export const kLogViewModelsTabId = "models";
+export const kLogViewTaskTabId = "task";
 
 // Sample tab constants
 export const kSampleMessagesTabId = `messages`;

inspect_ai/_view/www/src/utils/numeric.ts

@@ -0,0 +1,17 @@
+export function compareWithNan(a: number, b: number): number {
+  const aIsNaN = Number.isNaN(a);
+  const bIsNaN = Number.isNaN(b);
+
+  if (aIsNaN && bIsNaN) {
+    return 0;
+  }
+
+  if (aIsNaN) {
+    return 1;
+  }
+  if (bIsNaN) {
+    return -1;
+  }
+
+  return a - b;
+}

inspect_ai/agent/_agent.py

@@ -27,13 +27,14 @@ from inspect_ai.model._chat_message import (
     ChatMessageAssistant,
 )
 from inspect_ai.model._model_output import ChatCompletionChoice, ModelOutput
+from inspect_ai.util._limited_conversation import ChatMessageList
 
 
 class AgentState:
     """Agent state."""
 
     def __init__(self, *, messages: list[ChatMessage]) -> None:
-        self._messages = messages
+        self._messages: list[ChatMessage] = ChatMessageList(messages)
         self._output: ModelOutput | None = None
 
     @property
@@ -43,8 +44,7 @@ class AgentState:
 
     @messages.setter
     def messages(self, messages: list[ChatMessage]) -> None:
-        """Set the conversation history."""
-        self._messages = messages
+        self._messages = ChatMessageList(messages)
 
     @property
     def output(self) -> ModelOutput:

inspect_ai/agent/_as_solver.py

@@ -2,6 +2,8 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING, Any
 
+from inspect_ai.util._limit import Limit, apply_limits
+
 if TYPE_CHECKING:
     from inspect_ai.solver._solver import Solver
 
@@ -14,7 +16,7 @@ from inspect_ai.tool._tool_info import parse_tool_info
 from ._agent import Agent, AgentState
 
 
-def as_solver(agent: Agent, **agent_kwargs: Any) -> Solver:
+def as_solver(agent: Agent, limits: list[Limit] = [], **agent_kwargs: Any) -> Solver:
     """Convert an agent to a solver.
 
     Note that agents used as solvers will only receive their first parameter
@@ -23,6 +25,8 @@ def as_solver(agent: Agent, **agent_kwargs: Any) -> Solver:
 
     Args:
        agent: Agent to convert.
+       limits: List of limits to apply to the agent. Should a limit
+          be exceeded, the Sample ends and proceeds to scoring.
        **agent_kwargs: Arguments to curry to Agent function (required
           if the agent has parameters without default values).
 
@@ -52,17 +56,21 @@ def as_solver(agent: Agent, **agent_kwargs: Any) -> Solver:
     @solver(name=agent_name)
     def agent_to_solver() -> Solver:
         async def solve(state: TaskState, generate: Generate) -> TaskState:
-            # run agent
-            agent_state = await agent(
-                AgentState(messages=state.messages), **agent_kwargs
-            )
-
-            # update messages
-            state.messages = agent_state.messages
-
-            # update output if its not empty
-            if agent_state.output:
-                state.output = agent_state.output
+            agent_state = AgentState(messages=state.messages)
+
+            try:
+                # run the agent with limits
+                with apply_limits(limits):
+                    agent_state = await agent(agent_state, **agent_kwargs)
+            # if an exception occurs, we still want to update the TaskState with the
+            # AgentState's messages + output so that it appears in the log and is scored
+            finally:
+                # update messages
+                state.messages = agent_state.messages
+
+                # update output if its not empty
+                if agent_state.output:
+                    state.output = agent_state.output
 
             return state
 

inspect_ai/agent/_as_tool.py

@@ -10,12 +10,18 @@ from inspect_ai.tool._tool import Tool, ToolResult, tool
 from inspect_ai.tool._tool_def import ToolDef, validate_tool_parameters
 from inspect_ai.tool._tool_info import ToolInfo, parse_tool_info
 from inspect_ai.tool._tool_params import ToolParam
+from inspect_ai.util._limit import Limit, apply_limits
 
 from ._agent import AGENT_DESCRIPTION, Agent, AgentState
 
 
 @tool
-def as_tool(agent: Agent, description: str | None = None, **agent_kwargs: Any) -> Tool:
+def as_tool(
+    agent: Agent,
+    description: str | None = None,
+    limits: list[Limit] = [],
+    **agent_kwargs: Any,
+) -> Tool:
     """Convert an agent to a tool.
 
     By default the model will see all of the agent's arguments as
@@ -27,6 +33,9 @@ def as_tool(agent: Agent, description: str | None = None, **agent_kwargs: Any) -
     Args:
        agent: Agent to convert.
        description: Tool description (defaults to agent description)
+       limits: List of limits to apply to the agent. Should a limit
+          be exceeded, the tool call ends and returns an error
+          explaining that a limit was exceeded.
        **agent_kwargs: Arguments to curry to Agent function (arguments
           provided here will not be presented to the model as part
           of the tool interface).
@@ -41,9 +50,12 @@ def as_tool(agent: Agent, description: str | None = None, **agent_kwargs: Any) -
         )
 
     async def execute(input: str, *args: Any, **kwargs: Any) -> ToolResult:
-        # prepare state and call agent
+        # prepare state
         state = AgentState(messages=[ChatMessageUser(content=input, source="input")])
-        state = await agent(state, *args, **(agent_kwargs | kwargs))
+
+        # run the agent with limits
+        with apply_limits(limits):
+            state = await agent(state, *args, **(agent_kwargs | kwargs))
 
         # find assistant message to read content from (prefer output)
         if not state.output.empty:

inspect_ai/agent/_handoff.py

@@ -9,6 +9,7 @@ from inspect_ai._util.registry import (
 from inspect_ai.tool._tool import Tool, ToolResult, ToolSource
 from inspect_ai.tool._tool_def import ToolDef
 from inspect_ai.tool._tool_description import ToolDescription, set_tool_description
+from inspect_ai.util._limit import Limit
 
 from ._agent import Agent
 from ._as_tool import agent_tool_info
@@ -21,6 +22,7 @@ def handoff(
     input_filter: MessageFilter | None = None,
     output_filter: MessageFilter | None = None,
     tool_name: str | None = None,
+    limits: list[Limit] = [],
     **agent_kwargs: Any,
 ) -> Tool:
     """Create a tool that enables models to handoff to agents.
@@ -35,6 +37,9 @@ def handoff(
             Use the built-in `last_message` filter to return only the last message
             or alternatively specify a custom `MessageFilter` function.
         tool_name: Alternate tool name (defaults to `transfer_to_{agent_name}`)
+        limits: List of limits to apply to the agent. Should a limit be exceeded,
+            the agent stops and a user message is appended explaining that a limit was
+            exceeded.
         **agent_kwargs: Arguments to curry to `Agent` function (arguments provided here
             will not be presented to the model as part of the tool interface).
 
@@ -52,7 +57,7 @@ def handoff(
     tool_info = agent_tool_info(agent, description, **agent_kwargs)
 
     # AgentTool calls will be intercepted by execute_tools
-    agent_tool = AgentTool(agent, input_filter, output_filter, **agent_kwargs)
+    agent_tool = AgentTool(agent, input_filter, output_filter, limits, **agent_kwargs)
     tool_name = tool_name or f"transfer_to_{tool_info.name}"
     set_registry_info(agent_tool, RegistryInfo(type="tool", name=tool_name))
     set_tool_description(
@@ -72,11 +77,13 @@ class AgentTool(Tool):
         agent: Agent,
         input_filter: MessageFilter | None = None,
         output_filter: MessageFilter | None = None,
+        limits: list[Limit] = [],
         **kwargs: Any,
     ):
         self.agent = agent
         self.input_filter = input_filter
         self.output_filter = output_filter
+        self.limits = limits
         self.kwargs = kwargs
 
     @property

inspect_ai/agent/_run.py

@@ -2,12 +2,16 @@ from copy import copy
 from typing import Any
 
 from inspect_ai.model._chat_message import ChatMessage, ChatMessageUser
+from inspect_ai.util._limit import Limit, apply_limits
 
 from ._agent import Agent, AgentState
 
 
 async def run(
-    agent: Agent, input: str | list[ChatMessage] | AgentState, **agent_kwargs: Any
+    agent: Agent,
+    input: str | list[ChatMessage] | AgentState,
+    limits: list[Limit] = [],
+    **agent_kwargs: Any,
 ) -> AgentState:
     """Run an agent.
 
@@ -17,6 +21,9 @@ async def run(
     Args:
         agent: Agent to run.
         input: Agent input (string, list of messages, or an `AgentState`).
+        limits: List of limits to apply to the agent. Should a limit be
+            exceeded, a LimitExceededError is raised which the caller may
+            handle as appropriate.
         **agent_kwargs: Additional arguments to pass to agent.
 
     Returns:
@@ -43,5 +50,6 @@ async def run(
     # create state
     state = AgentState(messages=input_messages)
 
-    # run the agent
-    return await agent(state, **agent_kwargs)
+    # run the agent with limits
+    with apply_limits(limits):
+        return await agent(state, **agent_kwargs)

inspect_ai/log/__init__.py

@@ -9,6 +9,7 @@ from ._file import (
     read_eval_log,
     read_eval_log_async,
     read_eval_log_sample,
+    read_eval_log_sample_summaries,
     read_eval_log_samples,
     write_eval_log,
     write_eval_log_async,
@@ -28,6 +29,7 @@ from ._log import (
     EvalSampleLimit,
     EvalSampleReductions,
     EvalSampleScore,
+    EvalSampleSummary,
     EvalScore,
     EvalSpec,
     EvalStats,
@@ -70,6 +72,7 @@ __all__ = [
     "EvalSampleLimit",
     "EvalSampleScore",
     "EvalSampleReductions",
+    "EvalSampleSummary",
     "EvalScore",
     "EvalSpec",
     "EvalStats",
@@ -100,6 +103,7 @@ __all__ = [
     "read_eval_log_async",
     "read_eval_log_sample",
     "read_eval_log_samples",
+    "read_eval_log_sample_summaries",
     "condense_sample",
     "resolve_sample_attachments",
     "write_eval_log",