inspect-ai 0.3.63__py3-none-any.whl → 0.3.65__py3-none-any.whl

inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx

@@ -1,6 +1,7 @@
 import clsx from "clsx";
 import { EvalMetric, EvalResults, EvalScore, Reducer } from "../../types/log";
 import { formatPrettyDecimal } from "../../utils/format";
+import { metricDisplayName } from "../utils";
 import styles from "./ResultsPanel.module.css";
 
 interface ResultsPanelProps {
@@ -23,7 +24,7 @@ export const ResultsPanel: React.FC<ResultsPanelProps> = ({ results }) => {
           metric: {
             name: key,
             value: score.metrics[key].value,
-            options: {},
+            params: score.metrics[key].params,
             metadata: {},
           },
         };
@@ -31,18 +32,35 @@ export const ResultsPanel: React.FC<ResultsPanelProps> = ({ results }) => {
     });
 
     const metrics = Object.values(scorers)[0];
+    const showReducer = metrics && metrics.length > 0 && !!metrics[0].reducer;
     return (
       <div className={styles.simpleMetricsRows}>
         {metrics.map((metric, i) => {
-          return <VerticalMetric metricSummary={metric} isFirst={i === 0} />;
+          return (
+            <VerticalMetric
+              key={`simple-metric-${i}`}
+              metricSummary={metric}
+              isFirst={i === 0}
+              showReducer={showReducer}
+            />
+          );
         })}
       </div>
     );
   } else {
+    const showReducer =
+      results?.scores.findIndex((score) => !!score.reducer) !== -1;
     return (
       <div className={styles.multiMetricsRows}>
         {results?.scores?.map((score, index) => {
-          return <MultiScorerMetric scorer={score} isFirst={index === 0} />;
+          return (
+            <MultiScorerMetric
+              key={`multi-metric-${index}`}
+              scorer={score}
+              isFirst={index === 0}
+              showReducer={showReducer}
+            />
+          );
         })}
       </div>
     );
@@ -52,6 +70,7 @@ export const ResultsPanel: React.FC<ResultsPanelProps> = ({ results }) => {
 interface VerticalMetricProps {
   metricSummary: MetricSummary;
   isFirst: boolean;
+  showReducer: boolean;
 }
 
 /** Renders a Vertical Metric
@@ -59,21 +78,8 @@ interface VerticalMetricProps {
 const VerticalMetric: React.FC<VerticalMetricProps> = ({
   metricSummary,
   isFirst,
+  showReducer,
 }) => {
-  const reducer_component = metricSummary.reducer ? (
-    <div
-      className={clsx(
-        "text-style-label",
-        "text-style-secondary",
-        styles.verticalMetricReducer,
-      )}
-    >
-      {metricSummary.reducer}
-    </div>
-  ) : (
-    ""
-  );
-
   return (
     <div style={{ paddingLeft: isFirst ? "0" : "1em" }}>
       <div
@@ -84,11 +90,26 @@ const VerticalMetric: React.FC<VerticalMetricProps> = ({
           styles.verticalMetricName,
         )}
       >
-        {metricSummary.metric.name}
+        {metricDisplayName(metricSummary.metric)}
       </div>
-      {reducer_component}
+      {showReducer ? (
+        <div
+          className={clsx(
+            "text-style-label",
+            "text-style-secondary",
+            styles.verticalMetricReducer,
+          )}
+        >
+          {metricSummary.reducer || "default"}
+        </div>
+      ) : undefined}
+
       <div
-        className={clsx("vertical-metric-value", styles.verticalMetricValue)}
+        className={clsx(
+          "vertical-metric-value",
+          "text-size-largest",
+          styles.verticalMetricValue,
+        )}
       >
         {formatPrettyDecimal(metricSummary.metric.value)}
       </div>
@@ -99,33 +120,25 @@ const VerticalMetric: React.FC<VerticalMetricProps> = ({
 interface MultiScorerMetricProps {
   scorer: EvalScore;
   isFirst: boolean;
+  showReducer: boolean;
 }
 
 const MultiScorerMetric: React.FC<MultiScorerMetricProps> = ({
   scorer,
   isFirst,
+  showReducer,
 }) => {
   const titleFontClz = "text-size-base";
   const reducerFontClz = "text-size-smaller";
   const valueFontClz = "text-size-base";
 
-  const reducer_component = scorer.reducer ? (
+  return (
     <div
       className={clsx(
-        reducerFontClz,
-        "text-style-label",
-        "text-style-secondary",
-        styles.multiScorerReducer,
+        styles.multiScorer,
+        isFirst ? styles.multiScorerIndent : undefined,
       )}
     >
-      {scorer.reducer}
-    </div>
-  ) : (
-    ""
-  );
-
-  return (
-    <div style={{ paddingLeft: isFirst ? "0" : "1.5em" }}>
       <div
         className={clsx(
           titleFontClz,
@@ -137,13 +150,24 @@ const MultiScorerMetric: React.FC<MultiScorerMetricProps> = ({
       >
         {scorer.name}
       </div>
-      {reducer_component}
+      {showReducer ? (
+        <div
+          className={clsx(
+            reducerFontClz,
+            "text-style-label",
+            "text-style-secondary",
+            styles.multiScorerReducer,
+          )}
+        >
+          {scorer.reducer || "default"}
+        </div>
+      ) : undefined}
       <div className={clsx(valueFontClz, styles.multiScorerValue)}>
         {Object.keys(scorer.metrics).map((key) => {
           const metric = scorer.metrics[key];
           return (
-            <div>
-              <div>{metric.name}</div>
+            <div className={styles.multiScoreMetricGrid} key={key}>
+              <div>{metricDisplayName(metric)}</div>
               <div className={styles.multiScorerValueContent}>
                 {formatPrettyDecimal(metric.value)}
               </div>

inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.tsx

@@ -53,6 +53,7 @@ export const SecondaryBar: React.FC<SecondaryBarProps> = ({
     size: "minmax(12%, auto)",
     value: (
       <LabeledValue
+        key="sb-dataset"
         label="Dataset"
         className={(styles.staticCol, "text-size-small")}
       >
@@ -71,6 +72,7 @@ export const SecondaryBar: React.FC<SecondaryBarProps> = ({
     size: "minmax(12%, auto)",
     value: (
       <LabeledValue
+        key="sb-scorer"
         label={label}
         className={clsx(
           styles.staticCol,
@@ -88,6 +90,7 @@ export const SecondaryBar: React.FC<SecondaryBarProps> = ({
       size: "minmax(12%, auto)",
       value: (
         <LabeledValue
+          key="sb-params"
           label="Config"
           className={clsx(styles.justifyRight, "text-size-small")}
         >
@@ -106,6 +109,7 @@ export const SecondaryBar: React.FC<SecondaryBarProps> = ({
       size: "minmax(12%, auto)",
       value: (
         <LabeledValue
+          key="sb-duration"
           label="Duration"
           className={clsx(styles.justifyRight, "text-size-small")}
         >

inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.tsx

@@ -2,17 +2,19 @@ import clsx from "clsx";
 import { EvalScore } from "../../types/log";
 import { formatPrettyDecimal } from "../../utils/format";
 
+import { metricDisplayName } from "../utils";
 import styles from "./SidebarScoreView.module.css";
 interface SidebarScoreProps {
   scorer: EvalScore;
 }
 
 export const SidebarScoreView: React.FC<SidebarScoreProps> = ({ scorer }) => {
+  const showReducer = !!scorer.reducer;
   return (
     <div className={styles.container}>
       {Object.keys(scorer.metrics).map((metric) => {
         return (
-          <div className={styles.metric}>
+          <div className={styles.metric} key={metric}>
             <div
               className={clsx(
                 "text-style-secondary",
@@ -21,11 +23,11 @@ export const SidebarScoreView: React.FC<SidebarScoreProps> = ({ scorer }) => {
                 styles.metricName,
               )}
             >
-              {scorer.metrics[metric].name}
+              {metricDisplayName(scorer.metrics[metric])}
             </div>
-            {scorer.reducer ? (
+            {showReducer ? (
               <div className={clsx("text-size-small", styles.metricReducer)}>
-                ${scorer.reducer}
+                {scorer.reducer || "default"}
               </div>
             ) : (
               ""

inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.tsx

@@ -2,6 +2,7 @@ import clsx from "clsx";
 import { Fragment } from "react";
 import { Scores } from "../../types/log";
 import { formatPrettyDecimal } from "../../utils/format";
+import { metricDisplayName } from "../utils";
 import styles from "./SidebarScoresView.module.css";
 
 interface SidebarScoresProps {
@@ -9,26 +10,34 @@ interface SidebarScoresProps {
 }
 
 export const SidebarScoresView: React.FC<SidebarScoresProps> = ({ scores }) => {
+  const showReducer = scores.findIndex((score) => !!score.reducer) !== -1;
   return (
     <div className={styles.container}>
-      {scores.map((score) => {
+      {scores.map((score, idx) => {
         const name = score.name;
         const reducer = score.reducer;
         return (
-          <div className={styles.scoreWrapper}>
+          <div className={styles.scoreWrapper} key={`scorer-${name}-${idx}`}>
             <div
               className={clsx(
                 "text-style-secondary",
-                "text-label",
+                "text-style-label",
                 "text-size-small",
                 styles.metricName,
               )}
             >
               {name}
             </div>
-            {reducer ? (
-              <div className={clsx("text-size-small", styles.metricReducer)}>
-                {reducer}
+            {showReducer ? (
+              <div
+                className={clsx(
+                  "text-size-small",
+                  "text-style-label",
+                  "text-style-secondary",
+                  styles.metricReducer,
+                )}
+              >
+                {reducer || "default"}
               </div>
             ) : (
               ""
@@ -38,14 +47,7 @@ export const SidebarScoresView: React.FC<SidebarScoresProps> = ({ scores }) => {
                 const metric = score.metrics[key];
                 return (
                   <Fragment key={key}>
-                    <div
-                      className={clsx(
-                        "text-style-secondary",
-                        "text-style-label",
-                      )}
-                    >
-                      {metric.name}
-                    </div>
+                    <div className={clsx()}>{metricDisplayName(metric)}</div>
                     <div className={styles.metricValue}>
                       {formatPrettyDecimal(metric.value)}
                     </div>

inspect_ai/_view/www/src/workspace/tabs/InfoTab.tsx

@@ -36,24 +36,6 @@ export const InfoTab: React.FC<PlanTabProps> = ({
     setHidden(false);
   }, [evalSpec, evalPlan, evalResults, evalStats, samples]);
 
-  const infoCards = [];
-  infoCards.push([
-    <PlanCard
-      evalSpec={evalSpec}
-      evalPlan={evalPlan}
-      scores={evalResults?.scores}
-    />,
-  ]);
-
-  if (evalStatus !== "started") {
-    infoCards.push(<UsageCard stats={evalStats} />);
-  }
-
-  // If there is error or progress, includes those within info
-  if (evalStatus === "error" && evalError) {
-    infoCards.unshift(<TaskErrorCard error={evalError} />);
-  }
-
   const showWarning =
     (!samples || samples.length === 0) &&
     evalStatus === "success" &&
@@ -73,7 +55,15 @@ export const InfoTab: React.FC<PlanTabProps> = ({
         ""
       )}
       <div style={{ padding: "0.5em 1em 0 1em", width: "100%" }}>
-        {infoCards}
+        <PlanCard
+          evalSpec={evalSpec}
+          evalPlan={evalPlan}
+          scores={evalResults?.scores}
+        />
+        {evalStatus !== "started" ? <UsageCard stats={evalStats} /> : undefined}
+        {evalStatus === "error" && evalError ? (
+          <TaskErrorCard error={evalError} />
+        ) : undefined}
       </div>
     </div>
   );

inspect_ai/_view/www/src/workspace/utils.ts

@@ -0,0 +1,34 @@
+import { EvalMetric } from "../types/log";
+
+export const metricDisplayName = (metric: EvalMetric): string => {
+  let modifier = undefined;
+  for (const metricModifier of metricModifiers) {
+    modifier = metricModifier(metric);
+    if (modifier) {
+      break;
+    }
+  }
+  const metricName = !modifier ? metric.name : `${metric.name}[${modifier}]`;
+
+  return metricName;
+};
+
+type MetricModifier = (metric: EvalMetric) => string | undefined;
+
+const clusterMetricModifier: MetricModifier = (
+  metric: EvalMetric,
+): string | undefined => {
+  if (metric.name !== "stderr") {
+    return undefined;
+  }
+
+  const clusterValue = ((metric.params || {}) as Record<string, unknown>)[
+    "cluster"
+  ];
+  if (clusterValue === undefined || typeof clusterValue !== "string") {
+    return undefined;
+  }
+  return clusterValue;
+};
+
+const metricModifiers: MetricModifier[] = [clusterMetricModifier];

inspect_ai/approval/_approval.py

@@ -17,6 +17,8 @@ Possible values:
 
 
 class Approval(BaseModel):
+    """Approval details (decision, explanation, etc.)"""
+
     decision: ApprovalDecision
     """Approval decision."""
 

inspect_ai/approval/_approver.py

@@ -20,10 +20,10 @@ class Approver(Protocol):
         Approve or reject a tool call.
 
         Args:
-            message (str): Message genreated by the model along with the tool call.
-            call (ToolCall): The tool call to be approved.
-            view (ToolCallView): Custom rendering of tool context and call.
-            state (state | None): The current task state, if available.
+            message: Message genreated by the model along with the tool call.
+            call: The tool call to be approved.
+            view: Custom rendering of tool context and call.
+            state: The current task state, if available.
 
         Returns:
             Approval: An Approval object containing the decision and explanation.

inspect_ai/approval/_auto.py

@@ -11,7 +11,7 @@ def auto_approver(decision: ApprovalDecision = "approve") -> Approver:
     """Automatically apply a decision to tool calls.
 
     Args:
-       decision (ApprovalDecision): Decision to apply.
+       decision: Decision to apply.
 
     Returns:
        Approver: Auto approver.

inspect_ai/approval/_human/approver.py

@@ -14,6 +14,9 @@ def human_approver(
 ) -> Approver:
     """Interactive human approver.
 
+    Args:
+       choices: Choices to present to human.
+
     Returns:
        Approver: Interactive human approver.
     """

inspect_ai/approval/_policy.py

@@ -20,8 +20,13 @@ from ._call import call_approver, record_approval
 
 @dataclass
 class ApprovalPolicy:
+    """Policy mapping approvers to tools."""
+
     approver: Approver
+    """Approver for policy."""
+
     tools: str | list[str]
+    """Tools to use this approver for (can be full tool names or globs)."""
 
 
 def policy_approver(policies: str | list[ApprovalPolicy]) -> Approver:

inspect_ai/approval/_registry.py

@@ -31,11 +31,11 @@ def approver(*args: Any, name: str | None = None, **attribs: Any) -> Any:
     Args:
       *args: Function returning `Approver` targeted by
         plain approver decorator without attributes (e.g. `@approver`)
-      name (str | None):
+      name:
         Optional name for approver. If the decorator has no name
         argument then the name of the function
         will be used to automatically assign a name.
-      **attribs: (dict[str,Any]): Additional approver attributes.
+      **attribs: Additional approver attributes.
 
     Returns:
         Approver with registry attributes.

inspect_ai/dataset/_dataset.py

@@ -27,6 +27,8 @@ MT = TypeVar("MT", bound=BaseModel)
 
 
 class Sample(BaseModel):
+    r"""Sample for an evaluation task."""
+
     def __init__(
         self,
         input: str | list[ChatMessage],
@@ -38,22 +40,22 @@ class Sample(BaseModel):
         files: dict[str, str] | None = None,
         setup: str | None = None,
     ) -> None:
-        r"""Sample to be used in an evaluation task.
+        r"""Create a Sample.
 
         Args:
-            input (str | list[ChatMessage]): The input to be submitted to the model.
-            choices (list[str] | None): Optional. List of available answer choices
-            (used only for multiple-choice evals).
-            target (str | list[str]): Optional. Ideal target output. May be a literal value
+            input: The input to be submitted to the model.
+            choices: Optional. List of available answer choices
+                (used only for multiple-choice evals).
+            target: Optional. Ideal target output. May be a literal value
                 or narrative text to be used by a model grader.
-            id (int | str | None): Optional. Unique identifier for sample.
-            metadata (dict[str,Any] | None): Optional. Arbitrary metadata associated with the sample.
-            sandbox (SandboxEnvironmentType | None): Sandbox environment type
-            (or optionally a str or tuple with a shorthand spec)
-            files (dict[str, str] | None): Optional. Files that go along with the sample (copied to
-            SandboxEnvironment). Files can be paths, inline text, or inline binary (base64 encoded data URL).
-            setup (str | None): Optional. Setup script to run for sample (run
-            within default SandboxEnvironment).
+            id: Optional. Unique identifier for sample.
+            metadata: Optional. Arbitrary metadata associated with the sample.
+                sandbox (SandboxEnvironmentType | None): Sandbox environment type (or optionally a str or tuple with a shorthand spec)
+            sandbox: Optional. Sandbox specification for this sample.
+            files: Optional. Files that go along with the sample (copied to
+                SandboxEnvironment). Files can be paths, inline text, or inline binary (base64 encoded data URL).
+            setup: Optional. Setup script to run for sample (run
+                within default SandboxEnvironment).
         """
         super().__init__(
             input=input,
@@ -144,14 +146,6 @@ class Dataset(Sequence[Sample], abc.ABC):
     @abc.abstractmethod
     def shuffled(self) -> bool: ...
 
-    @abc.abstractmethod
-    def shuffle_choices(self, seed: int | None = None) -> None:
-        """Shuffle the order of the choices with each sample.
-
-        Args:
-           seed: (int | None): Random seed for shuffling (optional).
-        """
-
     @overload
     def __getitem__(self, index: int) -> Sample: ...
 
@@ -164,14 +158,6 @@ class Dataset(Sequence[Sample], abc.ABC):
     @abc.abstractmethod
     def __len__(self) -> int: ...
 
-    @abc.abstractmethod
-    def shuffle(self, seed: int | None = None) -> None:
-        """Shuffle the order of the dataset (in place).
-
-        Args:
-           seed: (int | None): Random seed for shuffling (optional).
-        """
-
     @abc.abstractmethod
     def sort(
         self,
@@ -185,8 +171,8 @@ class Dataset(Sequence[Sample], abc.ABC):
         The key function defaults to measuring the length of the sample's input field.
 
         Args:
-            reverse (bool): if true, sort in descending order. Defaults to False.
-            key (Callable[[Any], Any]): a callable mapping each item to a numeric value (optional, defaults to sample_input_len).
+            reverse: If `Treu`, sort in descending order. Defaults to False.
+            key: a callable mapping each item to a numeric value (optional, defaults to sample_input_len).
         """
 
     @abc.abstractmethod
@@ -196,28 +182,33 @@ class Dataset(Sequence[Sample], abc.ABC):
         """Filter the dataset using a predicate.
 
         Args:
-          predicate (Callable[[Sample], bool]): Filtering function.
-          name (str | None): Name for filtered dataset (optional).
+          predicate: Filtering function.
+          name: Name for filtered dataset (optional).
 
         Returns:
           Filtered dataset.
         """
 
+    @abc.abstractmethod
+    def shuffle(self, seed: int | None = None) -> None:
+        """Shuffle the order of the dataset (in place).
+
+        Args:
+           seed: Random seed for shuffling (optional).
+        """
+
+    @abc.abstractmethod
+    def shuffle_choices(self, seed: int | None = None) -> None:
+        """Shuffle the order of the choices with each sample.
+
+        Args:
+           seed: Random seed for shuffling (optional).
+        """
+
 
 @dataclass
 class FieldSpec:
-    r"""Specification for mapping data source fields to sample fields.
-
-    Args:
-        input (str): Name of the field containing the sample input.
-        target (str): Name of the field containing the sample target.
-        choices (str): Optional. Name of field containing the list of answer choices.
-        id (str): Optional. Unique identifier for the sample.
-        metadata (list[str] | None): List of additional field names that should be read as metadata.
-        sandbox (str): Optional. Sandbox type along with optional config file
-        files (str): Optional. Files that go along with the sample.
-        setup (str): Optional. Setup script to run for sample .
-    """
+    r"""Specification for mapping data source fields to sample fields."""
 
     input: str = field(default="input")
     """Name of the field containing the sample input."""

inspect_ai/dataset/_sources/csv.py

@@ -35,30 +35,30 @@ def csv_dataset(
     r"""Read dataset from CSV file.
 
     Args:
-        csv_file (str): Path to CSV file. Can be a local filesystem path,
+        csv_file: Path to CSV file. Can be a local filesystem path,
             a path to an S3 bucket (e.g. "s3://my-bucket"), or an HTTPS URL.
             Use `fs_options` to pass arguments through to the `S3FileSystem` constructor.
-        sample_fields (FieldSpec | RecordToSample): Method of mapping underlying
+        sample_fields: Method of mapping underlying
             fields in the data source to Sample objects. Pass `None` if the data is already
             stored in `Sample` form (i.e. has "input" and "target" columns.); Pass a
             `FieldSpec` to specify mapping fields by name; Pass a `RecordToSample` to
             handle mapping with a custom function that returns one or more samples.
-        auto_id (bool): Assign an auto-incrementing ID for each sample.
-        shuffle (bool): Randomly shuffle the dataset order.
-        seed: (int | None): Seed used for random shuffle.
-        shuffle_choices: (bool | int | None): Whether to shuffle the choices. If an int is passed, this will be used as the seed when shuffling.
-        limit (int | None): Limit the number of records to read.
-        dialect (str): CSV dialect ("unix", "excel" or"excel-tab"). Defaults to "unix". See https://docs.python.org/3/library/csv.html#dialects-and-formatting-parameters for more details
-        encoding (str): Text encoding for file (defaults to "utf-8").
-        name (str): Optional name for dataset (for logging). If not specified,
+        auto_id: Assign an auto-incrementing ID for each sample.
+        shuffle: Randomly shuffle the dataset order.
+        seed: Seed used for random shuffle.
+        shuffle_choices: Whether to shuffle the choices. If an int is passed, this will be used as the seed when shuffling.
+        limit: Limit the number of records to read.
+        dialect: CSV dialect ("unix", "excel" or"excel-tab"). Defaults to "unix". See https://docs.python.org/3/library/csv.html#dialects-and-formatting-parameters for more details
+        encoding: Text encoding for file (defaults to "utf-8").
+        name: Optional name for dataset (for logging). If not specified,
             defaults to the stem of the filename
-        fs_options (dict[str, Any]): Optional. Additional arguments to pass through
+        fs_options: Optional. Additional arguments to pass through
             to the filesystem provider (e.g. `S3FileSystem`). Use `{"anon": True }`
             if you are accessing a public S3 bucket with no credentials.
-        fieldnames (list[str] | None): Optional. A list of fieldnames to use for the CSV.
+        fieldnames: Optional. A list of fieldnames to use for the CSV.
             If None, the values in the first row of the file will be used as the fieldnames.
             Useful for files without a header.
-        delimiter (str): Optional. The delimiter to use when parsing the file. Defaults to ",".
+        delimiter: Optional. The delimiter to use when parsing the file. Defaults to ",".
 
     Returns:
         Dataset read from CSV file.