inspect-ai 0.3.68__py3-none-any.whl → 0.3.70__py3-none-any.whl

inspect_ai/_view/www/src/samples/SampleDisplay.tsx

@@ -24,6 +24,7 @@ import {
 } from "../constants";
 import { EvalSample } from "../types/log";
 import { ModelTokenTable } from "../usage/ModelTokenTable";
+import { formatTime } from "../utils/format";
 import { printHeadingHtml, printHtml } from "../utils/print";
 import { ChatViewVirtualList } from "./chat/ChatViewVirtualList";
 import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
@@ -180,7 +181,7 @@ export const SampleDisplay: React.FC<SampleDisplayProps> = ({
         {sampleMetadatas.length > 0 ? (
           <TabPanel
             id={kSampleMetdataTabId}
-            className="sample-tab"
+            className={clsx("sample-tab")}
             title="Metadata"
             onSelected={onSelectedTab}
             selected={selectedTab === kSampleMetdataTabId}
@@ -228,6 +229,7 @@ export const SampleDisplay: React.FC<SampleDisplayProps> = ({
 
 const metadataViewsForSample = (id: string, sample: EvalSample) => {
   const sampleMetadatas = [];
+
   if (sample.model_usage && Object.keys(sample.model_usage).length > 0) {
     sampleMetadatas.push(
       <Card key={`sample-usage-${id}`}>
@@ -242,6 +244,31 @@ const metadataViewsForSample = (id: string, sample: EvalSample) => {
     );
   }
 
+  if (
+    sample.total_time !== undefined &&
+    sample.total_time !== null &&
+    sample.working_time !== undefined &&
+    sample.working_time !== null
+  ) {
+    sampleMetadatas.push(
+      <Card key={`sample-time-${id}`}>
+        <CardHeader label="Time" />
+        <CardBody>
+          <div className={clsx(styles.timePanel, "text-size-smaller")}>
+            <div className={clsx("text-style-label", "text-style-secondary")}>
+              Working
+            </div>
+            <div>{formatTime(sample.working_time)}</div>
+            <div className={clsx("text-style-label", "text-style-secondary")}>
+              Total
+            </div>
+            <div>{formatTime(sample.total_time)}</div>
+          </div>
+        </CardBody>
+      </Card>,
+    );
+  }
+
   if (Object.keys(sample?.metadata).length > 0) {
     sampleMetadatas.push(
       <Card key={`sample-metadata-${id}`}>

inspect_ai/_view/www/src/samples/SampleSummaryView.module.css

@@ -22,3 +22,7 @@
 .wrap {
   word-wrap: anywhere;
 }
+
+.titled:hover {
+  cursor: pointer;
+}

inspect_ai/_view/www/src/samples/SampleSummaryView.tsx

@@ -1,7 +1,7 @@
 import clsx from "clsx";
 import { MarkdownDiv } from "../components/MarkdownDiv";
-import { EvalSample } from "../types/log";
-import { arrayToString, inputString } from "../utils/format";
+import { EvalSample, WorkingTime } from "../types/log";
+import { arrayToString, formatTime, inputString } from "../utils/format";
 import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
 import { FlatSampleError } from "./error/FlatSampleErrorView";
 
@@ -20,6 +20,7 @@ interface SummaryColumn {
   size: string;
   center?: boolean;
   clamp?: boolean;
+  title?: string;
 }
 
 /**
@@ -46,6 +47,7 @@ export const SampleSummaryView: React.FC<SampleSummaryViewProps> = ({
     sampleDescriptor?.messageShape.normalized.limit > 0
       ? Math.max(0.15, sampleDescriptor.messageShape.normalized.limit)
       : 0;
+  const timeSize = sample.working_time || sample.total_time ? 0.15 : 0;
   const idSize = Math.max(
     2,
     Math.min(10, sampleDescriptor?.messageShape.raw.id),
@@ -110,6 +112,23 @@ export const SampleSummaryView: React.FC<SampleSummaryViewProps> = ({
     });
   }
 
+  const toolTip = (working_time?: WorkingTime) => {
+    if (working_time === undefined || working_time === null) {
+      return undefined;
+    }
+    return `Working time: ${formatTime(working_time)}`;
+  };
+
+  if (sample.total_time) {
+    columns.push({
+      label: "Time",
+      value: formatTime(sample.total_time),
+      size: `${timeSize}fr`,
+      center: true,
+      title: toolTip(sample.working_time),
+    });
+  }
+
   if (sample?.limit && limitSize > 0) {
     columns.push({
       label: "Limit",
@@ -151,8 +170,10 @@ export const SampleSummaryView: React.FC<SampleSummaryViewProps> = ({
               "text-style-label",
               "text-style-secondary",
               "text-size-base",
+              col.title ? styles.titled : undefined,
               col.center ? styles.centerLabel : undefined,
             )}
+            title={col.title}
           >
             {col.label}
           </div>

inspect_ai/_view/www/src/samples/descriptor/score/ObjectScoreDescriptor.tsx

@@ -49,7 +49,7 @@ export const objectScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
                   ? value
                   : parseFloat(value === true ? "1" : value),
               )
-            : value;
+            : String(value);
         scores.push(
           <div
             className={clsx(

inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx

@@ -33,6 +33,8 @@ export const SampleLimitEventView: React.FC<SampleLimitEventViewProps> = ({
         return "Token Limit Exceeded";
       case "operator":
         return "Operator Canceled";
+      case "working":
+        return "Execution Time Limit Exceeded";
     }
   };
 
@@ -48,6 +50,8 @@ export const SampleLimitEventView: React.FC<SampleLimitEventViewProps> = ({
         return ApplicationIcons.limits.tokens;
       case "operator":
         return ApplicationIcons.limits.operator;
+      case "working":
+        return ApplicationIcons.limits.execution;
     }
   };
 

inspect_ai/_view/www/src/samples/transcript/SandboxEventView.module.css

@@ -0,0 +1,32 @@
+.contents {
+  margin-top: 0.5em;
+}
+
+.contents > :last-child {
+  margin-bottom: 0;
+}
+
+.twoColumn {
+  display: grid;
+  grid-template-columns: auto 1fr;
+  column-gap: 1.5em;
+}
+
+.exec {
+  margin-top: 0.5em;
+}
+
+.result {
+  margin-top: 0.5em;
+}
+
+.fileLabel {
+  margin-top: 0;
+  margin-bottom: 0;
+}
+
+.wrapPre {
+  white-space: pre-wrap;
+  word-wrap: break-word;
+  overflow-wrap: break-word;
+}

inspect_ai/_view/www/src/samples/transcript/SandboxEventView.tsx

@@ -0,0 +1,152 @@
+import { ApplicationIcons } from "../../appearance/icons";
+import ExpandablePanel from "../../components/ExpandablePanel";
+import { MarkdownDiv } from "../../components/MarkdownDiv";
+import { MetaDataGrid } from "../../metadata/MetaDataGrid";
+import { SandboxEvent } from "../../types/log";
+import { formatDateTime } from "../../utils/format";
+import { EventPanel } from "./event/EventPanel";
+import { EventSection } from "./event/EventSection";
+import { TranscriptEventState } from "./types";
+
+import clsx from "clsx";
+import styles from "./SandboxEventView.module.css";
+
+interface SandboxEventViewProps {
+  id: string;
+  event: SandboxEvent;
+  eventState: TranscriptEventState;
+  setEventState: (state: TranscriptEventState) => void;
+  className?: string | string[];
+}
+
+/**
+ * Renders the SandboxEventView component.
+ */
+export const SandboxEventView: React.FC<SandboxEventViewProps> = ({
+  id,
+  event,
+  eventState,
+  setEventState,
+  className,
+}) => {
+  return (
+    <EventPanel
+      id={id}
+      className={className}
+      title={`Sandbox: ${event.action}`}
+      icon={ApplicationIcons.sandbox}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      selectedNav={eventState.selectedNav || ""}
+      setSelectedNav={(selectedNav) => {
+        setEventState({ ...eventState, selectedNav });
+      }}
+      collapsed={eventState.collapsed}
+      setCollapsed={(collapsed) => {
+        setEventState({ ...eventState, collapsed });
+      }}
+    >
+      {event.action === "exec" ? (
+        <ExecView event={event} />
+      ) : event.action === "read_file" ? (
+        <ReadFileView event={event} />
+      ) : (
+        <WriteFileView event={event} />
+      )}
+    </EventPanel>
+  );
+};
+
+interface ExecViewProps {
+  event: SandboxEvent;
+}
+
+const ExecView: React.FC<ExecViewProps> = ({ event }) => {
+  if (event.cmd === null) {
+    return undefined;
+  }
+  const cmd = event.cmd;
+  const options = event.options;
+  const input = event.input;
+  const result = event.result;
+  const output = event.output;
+
+  return (
+    <div className={clsx(styles.exec)}>
+      <EventSection title={`Command`}>
+        <div className={clsx(styles.twoColumn)}>
+          <pre className={clsx(styles.wrapPre)}>{cmd}</pre>
+          <pre className={clsx(styles.wrapPre)}>
+            {input !== null ? input?.trim() : undefined}
+          </pre>
+
+          {options !== null ? (
+            <EventSection title={`Options`}>
+              <MetaDataGrid
+                entries={options as Record<string, unknown>}
+                plain={true}
+              />
+            </EventSection>
+          ) : undefined}
+        </div>
+      </EventSection>
+      <EventSection title={`Result`}>
+        {output ? (
+          <ExpandablePanel collapse={false}>
+            <MarkdownDiv markdown={output} />
+          </ExpandablePanel>
+        ) : undefined}
+        <div className={clsx(styles.result)}>Exited with code {result}</div>
+      </EventSection>
+    </div>
+  );
+};
+
+interface ReadFileViewProps {
+  event: SandboxEvent;
+}
+
+const ReadFileView: React.FC<ReadFileViewProps> = ({ event }) => {
+  if (event.file === null) {
+    return undefined;
+  }
+  const file = event.file;
+  const output = event.output;
+  return <FileView file={file} contents={output?.trim()} />;
+};
+
+interface WriteFileViewProps {
+  event: SandboxEvent;
+}
+
+const WriteFileView: React.FC<WriteFileViewProps> = ({ event }) => {
+  if (event.file === null) {
+    return undefined;
+  }
+  const file = event.file;
+  const input = event.input;
+
+  return <FileView file={file} contents={input?.trim()} />;
+};
+
+interface FileViewProps {
+  file: string;
+  contents?: string;
+}
+
+const FileView: React.FC<FileViewProps> = ({ file, contents }) => {
+  return (
+    <div>
+      <EventSection title="File">
+        <pre className={clsx(styles.fileLabel)}>{file}</pre>
+      </EventSection>
+
+      {contents ? (
+        <EventSection title="Contents">
+          <ExpandablePanel collapse={false}>
+            <pre>{contents}</pre>
+          </ExpandablePanel>
+        </EventSection>
+      ) : undefined}
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/transcript/StepEventView.tsx

@@ -47,7 +47,7 @@ export const StepEventView: React.FC<StepEventViewProps> = ({
       title={title}
       subTitle={formatDateTime(new Date(event.timestamp))}
       icon={descriptor.icon}
-      collapse={false}
+      collapse={descriptor.collapse}
       text={text}
       selectedNav={eventState.selectedNav || ""}
       setSelectedNav={(selectedNav) => {
@@ -115,7 +115,7 @@ const summarize = (children: EventNode[]) => {
  */
 const stepDescriptor = (
   event: StepEvent,
-): { icon?: string; name?: string; endSpace?: boolean } => {
+): { icon?: string; name?: string; endSpace?: boolean; collapse?: boolean } => {
   const rootStepDescriptor = {
     endSpace: true,
   };
@@ -161,6 +161,13 @@ const stepDescriptor = (
         return {
           ...rootStepDescriptor,
           name: "Sample Init",
+          collapse: true,
+        };
+      case "init":
+        return {
+          ...rootStepDescriptor,
+          name: "Init",
+          collapse: true,
         };
       default:
         return {

inspect_ai/_view/www/src/samples/transcript/TranscriptView.tsx

@@ -17,6 +17,7 @@ import { ToolEventView } from "./ToolEventView";
 import { EventNode, EventType, TranscriptEventState } from "./types";
 
 import clsx from "clsx";
+import { SandboxEventView } from "./SandboxEventView";
 import styles from "./TranscriptView.module.css";
 
 interface TranscriptViewProps {
@@ -373,6 +374,17 @@ export const RenderedEventNode: React.FC<RenderedEventNodeProps> = ({
     case "approval":
       return <ApprovalEventView event={node.event} className={className} />;
 
+    case "sandbox":
+      return (
+        <SandboxEventView
+          id={id}
+          event={node.event}
+          className={className}
+          eventState={eventState}
+          setEventState={setEventState}
+        />
+      );
+
     default:
       return null;
   }
@@ -390,8 +402,14 @@ const fixupEventStream = (events: Events) => {
   // Filter pending events
   const finalEvents = events.filter((e) => !e.pending);
 
+  // See if the find an init step
+  const hasInitStep =
+    events.findIndex((e) => {
+      return e.event === "step" && e.name === "init";
+    }) !== -1;
+
   const fixedUp = [...finalEvents];
-  if (initEvent) {
+  if (!hasInitStep && initEvent) {
     fixedUp.splice(initEventIndex, 0, {
       timestamp: initEvent.timestamp,
       event: "step",

inspect_ai/_view/www/src/samples/transcript/event/EventPanel.tsx

@@ -55,16 +55,19 @@ export const EventPanel: React.FC<EventPanelProps> = ({
   const defaultPillId = pillId(0);
 
   const gridColumns = [];
+
+  // chevron
   if (hasCollapse) {
     gridColumns.push("minmax(0, max-content)");
   }
+
+  // icon
   if (icon) {
     gridColumns.push("max-content");
   }
+
+  // title
   gridColumns.push("minmax(0, max-content)");
-  if (subTitle) {
-    gridColumns.push("minmax(0, max-content)");
-  }
   gridColumns.push("auto");
   gridColumns.push("minmax(0, max-content)");
   gridColumns.push("minmax(0, max-content)");

inspect_ai/_view/www/src/samples/transcript/types.ts

@@ -8,6 +8,7 @@ import {
   ModelEvent,
   SampleInitEvent,
   SampleLimitEvent,
+  SandboxEvent,
   ScoreEvent,
   StateEvent,
   StepEvent,
@@ -37,7 +38,8 @@ export type EventType =
   | ToolEvent
   | InputEvent
   | ErrorEvent
-  | ApprovalEvent;
+  | ApprovalEvent
+  | SandboxEvent;
 
 export class EventNode {
   event: EventType;