inspect-ai 0.3.92__py3-none-any.whl → 0.3.94__py3-none-any.whl

inspect_ai/_view/www/src/app/plan/ModelCard.tsx

@@ -3,7 +3,6 @@ import { FC } from "react";
 import clsx from "clsx";
 import { EvalModelConfig, EvalSpec } from "../../@types/log";
 import { Card, CardBody, CardHeader } from "../../components/Card";
-import { ApplicationIcons } from "../appearance/icons";
 import { MetaDataGrid } from "../content/MetaDataGrid";
 import styles from "./ModelCard.module.css";
 
@@ -33,7 +32,7 @@ export const ModelCard: FC<ModelCardProps> = ({ evalSpec }) => {
 
   return (
     <Card>
-      <CardHeader icon={ApplicationIcons.model} label="Models" />
+      <CardHeader label="Models" />
       <CardBody id={"task-model-card-body"}>
         <div className={styles.container}>
           {Object.keys(modelsInfo || {}).map((modelKey) => {

inspect_ai/_view/www/src/app/plan/PlanCard.tsx

@@ -1,7 +1,7 @@
 import { FC } from "react";
 import { EvalPlan, EvalScore, EvalSpec } from "../../@types/log";
 import { Card, CardBody, CardHeader } from "../../components/Card";
-import { ApplicationIcons } from "../appearance/icons";
+import { MetaDataView } from "../content/MetaDataView";
 import { PlanDetailView } from "./PlanDetailView";
 
 interface PlanCardProps {
@@ -14,12 +14,34 @@ interface PlanCardProps {
  * Renders the plan card
  */
 export const PlanCard: FC<PlanCardProps> = ({ evalSpec, evalPlan, scores }) => {
+  const metadata = evalSpec?.metadata || {};
+
   return (
-    <Card>
-      <CardHeader icon={ApplicationIcons.config} label="Config" />
-      <CardBody id={"task-plan-card-body"}>
-        <PlanDetailView evaluation={evalSpec} plan={evalPlan} scores={scores} />
-      </CardBody>
-    </Card>
+    <>
+      <Card>
+        <CardHeader label="Summary" />
+        <CardBody id={"task-plan-card-body"}>
+          <PlanDetailView
+            evaluation={evalSpec}
+            plan={evalPlan}
+            scores={scores}
+          />
+        </CardBody>
+      </Card>
+
+      {Object.keys(metadata).length > 0 && (
+        <Card>
+          <CardHeader label="Metadata" />
+          <CardBody id={"task-metadata`"}>
+            <MetaDataView
+              key={`plan-md-metadata`}
+              className={"text-size-small"}
+              entries={metadata}
+              tableOptions="sm"
+            />
+          </CardBody>
+        </Card>
+      )}
+    </>
   );
 };

inspect_ai/_view/www/src/app/plan/PlanDetailView.module.css

@@ -22,7 +22,7 @@
 }
 
 .planCol {
-  margin-top: em;
+  margin-top: 0;
 }
 
 .container {

inspect_ai/_view/www/src/app/plan/PlanDetailView.tsx

@@ -1,14 +1,10 @@
 import { EvalPlan, EvalScore, EvalSpec, Params2 } from "../../@types/log";
-import { toTitleCase } from "../../utils/format";
-import { ghCommitUrl } from "../../utils/git";
-import { MetaDataView } from "../content/MetaDataView";
 import { DatasetDetailView } from "./DatasetDetailView";
 import { ScorerDetailView } from "./ScorerDetailView";
 import { SolversDetailView } from "./SolverDetailView";
 
 import clsx from "clsx";
 import { FC, ReactNode } from "react";
-import { kModelNone } from "../../constants";
 import styles from "./PlanDetailView.module.css";
 
 interface PlanDetailViewProps {
@@ -26,71 +22,7 @@ export const PlanDetailView: FC<PlanDetailViewProps> = ({
     return null;
   }
 
-  // Add configuration
-  const config: Record<string, unknown> = {};
-  Object.entries(evaluation?.config || {}).forEach((entry) => {
-    const key = entry[0];
-    const value = entry[1];
-    config[key] = value;
-  });
-
   const steps = plan?.steps;
-  const metadata = evaluation?.metadata;
-  const revision = evaluation?.revision;
-  const packages = evaluation?.packages;
-  const model_args = evaluation?.model_args;
-  const task_args = evaluation?.task_args;
-  const generate_config = plan?.config;
-
-  const taskInformation: Record<string, unknown> = {
-    ["Task ID"]: evaluation?.task_id,
-    ["Run ID"]: evaluation?.run_id,
-  };
-  if (revision) {
-    taskInformation[
-      `${revision.type ? `${toTitleCase(revision.type)} ` : ""}Revision`
-    ] = {
-      _html: (
-        <a href={ghCommitUrl(revision.origin, revision.commit)}>
-          {revision.commit}
-        </a>
-      ),
-    };
-  }
-  if (packages) {
-    const names = Object.keys(packages).map((key) => {
-      return `${key} ${packages[key]}`;
-    });
-
-    if (names.length === 1) {
-      taskInformation["Inspect"] = names[0];
-    } else {
-      taskInformation["Inspect"] = names;
-    }
-  }
-  if (evaluation.tags) {
-    taskInformation["Tags"] = evaluation.tags.join(", ");
-  }
-
-  if (evaluation?.model && evaluation.model !== kModelNone) {
-    config["model"] = evaluation.model;
-  }
-
-  if (evaluation?.model_base_url) {
-    config["model_base_url"] = evaluation.model_base_url;
-  }
-
-  if (evaluation?.sandbox) {
-    if (Array.isArray(evaluation?.sandbox)) {
-      config["sandbox"] = evaluation.sandbox[0];
-      if (evaluation.sandbox[1]) {
-        config["sandbox_config"] = evaluation.sandbox[1];
-      }
-    } else {
-      config["sandbox"] = evaluation?.sandbox.type;
-      config["sandbox_config"] = evaluation?.sandbox.config;
-    }
-  }
 
   const taskColumns: {
     title: string;
@@ -148,117 +80,12 @@ export const PlanDetailView: FC<PlanDetailViewProps> = ({
     }
   }
 
-  // Compute the column style for the remaining (either 1 or 2 columns wide)
-  const metadataColumns: {
-    title: string;
-    className: string;
-    contents: ReactNode;
-  }[] = [];
-  const cols = colCount(
-    metadataColumns,
-    task_args,
-    model_args,
-    config,
-    metadata,
-  );
-
-  metadataColumns.push({
-    title: "Task Information",
-    className: cols === 1 ? styles.oneCol : styles.twoCol,
-    contents: (
-      <MetaDataView
-        key={`plan-md-task`}
-        className={"text-size-small"}
-        entries={taskInformation}
-        tableOptions="sm"
-      />
-    ),
-  });
-
-  if (task_args && Object.keys(task_args).length > 0) {
-    metadataColumns.push({
-      title: "Task Args",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-task-args`}
-          className={"text-size-small"}
-          entries={task_args as Record<string, unknown>}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-  if (model_args && Object.keys(model_args).length > 0) {
-    metadataColumns.push({
-      title: "Model Args",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-model-args`}
-          className={"text-size-small"}
-          entries={model_args as Record<string, unknown>}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-
-  if (config && Object.keys(config).length > 0) {
-    metadataColumns.push({
-      title: "Configuration",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-config`}
-          className={"text-size-small"}
-          entries={config}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-
-  if (generate_config && Object.keys(generate_config).length > 0) {
-    const generate_record: Record<string, unknown> = Object.fromEntries(
-      Object.entries(generate_config),
-    );
-
-    metadataColumns.push({
-      title: "Generate Config",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-generate-config`}
-          className={"text-size-small"}
-          entries={generate_record}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-
-  if (metadata && Object.keys(metadata).length > 0) {
-    metadataColumns.push({
-      title: "Metadata",
-      className: cols === 1 ? styles.oneCol : styles.twoCol,
-      contents: (
-        <MetaDataView
-          key={`plan-md-metadata`}
-          className={"text-size-small"}
-          entries={metadata}
-          tableOptions="sm"
-        />
-      ),
-    });
-  }
-
   return (
     <div className={styles.container}>
       <div
         className={styles.grid}
         style={{
-          gridTemplateColumns: `repeat(${taskColumns.length}, auto)`,
+          gridTemplateColumns: `repeat(${taskColumns.length}, fit-content(50%))`,
         }}
       >
         {taskColumns.map((col) => {
@@ -273,34 +100,10 @@ export const PlanDetailView: FC<PlanDetailViewProps> = ({
           );
         })}
       </div>
-
-      <div className={clsx(styles.row)}>
-        {metadataColumns.map((col) => {
-          return (
-            <PlanColumn
-              title={col.title}
-              className={col.className}
-              key={`plan-col-${col.title}`}
-            >
-              {col.contents}
-            </PlanColumn>
-          );
-        })}
-      </div>
     </div>
   );
 };
 
-const colCount = (...other: unknown[]) => {
-  let count = 0;
-  for (const o in other) {
-    if (o && Object.keys(o).length > 0) {
-      count++;
-    }
-  }
-  return count;
-};
-
 interface PlanColumnProps {
   title: string;
   className: string | string[];

inspect_ai/_view/www/src/app/samples/descriptor/score/NumericScoreDescriptor.tsx

@@ -1,6 +1,7 @@
 import { Value2 } from "../../../../@types/log";
 import { kScoreTypeNumeric } from "../../../../constants";
 import { formatDecimalNoTrailingZeroes } from "../../../../utils/format";
+import { compareWithNan } from "../../../../utils/numeric";
 import { ScoreDescriptor } from "../types";
 
 export const numericScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
@@ -14,7 +15,7 @@ export const numericScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
     max: Math.max(...onlyNumeric),
     compare: (a, b) => {
       if (typeof a.value === "number" && typeof b.value === "number") {
-        return a.value - b.value;
+        return compareWithNan(a.value, b.value);
       } else {
         console.warn("Comparing non-numerics using a numeric score descriptor");
         return 0;

inspect_ai/_view/www/src/app/samples/transcript/SandboxEventView.module.css

@@ -13,7 +13,7 @@
 }
 
 .exec {
-  margin-top: 0.5em;
+  margin-top: 0;
 }
 
 .result {
@@ -29,4 +29,5 @@
   white-space: pre-wrap;
   word-wrap: break-word;
   overflow-wrap: break-word;
+  margin-bottom: 0;
 }

inspect_ai/_view/www/src/app/samples/transcript/SpanEventView.tsx

@@ -0,0 +1,174 @@
+import clsx from "clsx";
+import { FC } from "react";
+import { SpanBeginEvent } from "../../../@types/log";
+import { formatDateTime } from "../../../utils/format";
+import { EventPanel } from "./event/EventPanel";
+import { TranscriptComponent } from "./TranscriptView";
+import { kSandboxSignalName } from "./transform/fixups";
+import { EventNode } from "./types";
+
+interface SpanEventViewProps {
+  id: string;
+  event: SpanBeginEvent;
+  children: EventNode[];
+  className?: string | string[];
+}
+
+/**
+ * Renders the SpanEventView component.
+ */
+export const SpanEventView: FC<SpanEventViewProps> = ({
+  id,
+  event,
+  children,
+  className,
+}) => {
+  const descriptor = spanDescriptor(event);
+  const title =
+    descriptor.name ||
+    `${event.type ? event.type + ": " : "Step: "}${event.name}`;
+  const text = summarize(children);
+
+  return (
+    <EventPanel
+      id={`span-${event.name}-${id}`}
+      className={clsx("transcript-span", className)}
+      title={title}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      text={text}
+      collapse={descriptor.collapse}
+      icon={descriptor.icon}
+    >
+      <TranscriptComponent
+        id={`span|${event.name}|${id}`}
+        eventNodes={children}
+      />
+    </EventPanel>
+  );
+};
+
+const summarize = (children: EventNode[]) => {
+  if (children.length === 0) {
+    return "(no events)";
+  }
+
+  const formatEvent = (event: string, count: number) => {
+    if (count === 1) {
+      return `${count} ${event} event`;
+    } else {
+      return `${count} ${event} events`;
+    }
+  };
+
+  // Count the types
+  const typeCount: Record<string, number> = {};
+  children.forEach((child) => {
+    const currentCount = typeCount[child.event.event] || 0;
+    typeCount[child.event.event] = currentCount + 1;
+  });
+
+  // Try to summarize event types
+  const numberOfTypes = Object.keys(typeCount).length;
+  if (numberOfTypes < 3) {
+    return Object.keys(typeCount)
+      .map((key) => {
+        return formatEvent(key, typeCount[key]);
+      })
+      .join(", ");
+  }
+
+  // To many types, just return the number of events
+  if (children.length === 1) {
+    return "1 event";
+  } else {
+    return `${children.length} events`;
+  }
+};
+
+/**
+ * Returns a descriptor object containing icon and style based on the event type and name.
+ */
+const spanDescriptor = (
+  event: SpanBeginEvent,
+): { icon?: string; name?: string; endSpace?: boolean; collapse?: boolean } => {
+  const rootStepDescriptor = {
+    endSpace: true,
+  };
+
+  if (event.type === "solver") {
+    switch (event.name) {
+      case "chain_of_thought":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "generate":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "self_critique":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "system_message":
+        return {
+          ...rootStepDescriptor,
+          collapse: true,
+        };
+      case "use_tools":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      case "multiple_choice":
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+      default:
+        return {
+          ...rootStepDescriptor,
+          collapse: false,
+        };
+    }
+  } else if (event.type === "scorer") {
+    return {
+      ...rootStepDescriptor,
+      collapse: false,
+    };
+  } else if (event.event === "span_begin") {
+    if (event.span_id === kSandboxSignalName) {
+      return {
+        ...rootStepDescriptor,
+        name: "Sandbox Events",
+        collapse: true,
+      };
+    } else if (event.name === "init") {
+      return {
+        ...rootStepDescriptor,
+        name: "Init",
+        collapse: true,
+      };
+    } else {
+      return {
+        ...rootStepDescriptor,
+        collapse: false,
+      };
+    }
+  } else {
+    switch (event.name) {
+      case "sample_init":
+        return {
+          ...rootStepDescriptor,
+          name: "Sample Init",
+          collapse: true,
+        };
+      default:
+        return {
+          endSpace: false,
+        };
+    }
+  }
+};

inspect_ai/_view/www/src/app/samples/transcript/ToolEventView.tsx

@@ -4,7 +4,7 @@ import { resolveToolInput } from "../chat/tools/tool";
 import { ToolCallView } from "../chat/tools/ToolCallView";
 import { ApprovalEventView } from "./ApprovalEventView";
 import { EventPanel } from "./event/EventPanel";
-import { TranscriptView } from "./TranscriptView";
+import { TranscriptComponent } from "./TranscriptView";
 
 import clsx from "clsx";
 import { FC, useMemo } from "react";
@@ -12,11 +12,12 @@ import { PulsingDots } from "../../../components/PulsingDots";
 import { ChatView } from "../chat/ChatView";
 import { formatTiming, formatTitle } from "./event/utils";
 import styles from "./ToolEventView.module.css";
+import { EventNode } from "./types";
 
 interface ToolEventViewProps {
   id: string;
   event: ToolEvent;
-  depth: number;
+  children: EventNode[];
   className?: string | string[];
 }
 
@@ -26,7 +27,7 @@ interface ToolEventViewProps {
 export const ToolEventView: FC<ToolEventViewProps> = ({
   id,
   event,
-  depth,
+  children,
   className,
 }) => {
   // Extract tool input
@@ -92,13 +93,12 @@ export const ToolEventView: FC<ToolEventViewProps> = ({
           </div>
         ) : undefined}
       </div>
-      {event.events.length > 0 ? (
-        <TranscriptView
-          id={`${id}-subtask`}
+      {children.length > 0 ? (
+        <TranscriptComponent
           data-name="Transcript"
+          id={`${id}-subtask`}
+          eventNodes={children}
           data-default={event.failed || event.agent ? true : null}
-          events={event.events}
-          depth={depth + 1}
         />
       ) : (
         ""

inspect_ai/_view/www/src/app/samples/transcript/TranscriptView.tsx

@@ -17,6 +17,7 @@ import { ToolEventView } from "./ToolEventView";
 import { EventNode } from "./types";
 
 import clsx from "clsx";
+import { SpanEventView } from "./SpanEventView";
 import styles from "./TranscriptView.module.css";
 import { TranscriptVirtualListComponent } from "./TranscriptVirtualListComponent";
 import { fixupEventStream } from "./transform/fixups";
@@ -64,7 +65,6 @@ export const TranscriptVirtualList: FC<TranscriptVirtualListProps> = memo(
     const eventNodes = useMemo(() => {
       const resolvedEvents = fixupEventStream(events, !running);
       const eventNodes = treeifyEvents(resolvedEvents, depth || 0);
-
       return eventNodes;
     }, [events, depth]);
 
@@ -201,6 +201,16 @@ export const RenderedEventNode: FC<RenderedEventNodeProps> = memo(
           <StateEventView id={id} event={node.event} className={className} />
         );
 
+      case "span_begin":
+        return (
+          <SpanEventView
+            id={id}
+            event={node.event}
+            children={node.children}
+            className={className}
+          />
+        );
+
       case "step":
         return (
           <StepEventView
@@ -237,7 +247,7 @@ export const RenderedEventNode: FC<RenderedEventNodeProps> = memo(
             id={id}
             event={node.event}
             className={className}
-            depth={node.depth}
+            children={node.children}
           />
         );
 

inspect_ai/_view/www/src/app/samples/transcript/TranscriptVirtualListComponent.module.css

@@ -8,7 +8,7 @@
 
 .node {
   padding-top: 0.7rem;
-  padding-bottom: 0em;
+  padding-bottom: 1px;
 }
 
 .attached {

inspect_ai/_view/www/src/app/samples/transcript/event/EventPanel.tsx

@@ -9,7 +9,6 @@ import {
 import { ApplicationIcons } from "../../../appearance/icons";
 import { EventNavs } from "./EventNavs";
 
-import { ProgressBar } from "../../../../components/ProgressBar";
 import { useProperty } from "../../../../state/hooks";
 import styles from "./EventPanel.module.css";
 
@@ -41,7 +40,6 @@ export const EventPanel: FC<EventPanelProps> = ({
   icon,
   collapse,
   children,
-  running,
 }) => {
   const [isCollapsed, setCollapsed] = useProperty(id, "collapsed", {
     defaultValue: !!collapse,
@@ -191,7 +189,6 @@ export const EventPanel: FC<EventPanelProps> = ({
           })}
         </div>
       </div>
-      <ProgressBar animating={!!running} />
     </>
   );
   return card;