inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/samples/transcript/SampleInitEventView.tsx

@@ -0,0 +1,112 @@
+// @ts-check
+import { ApplicationIcons } from "../../appearance/icons";
+import { MetaDataGrid } from "../../metadata/MetaDataGrid";
+import { Messages, SampleInitEvent } from "../../types/log";
+import { formatDateTime } from "../../utils/format";
+import { toArray } from "../../utils/type";
+import { ChatView } from "../chat/ChatView";
+import { EventPanel } from "./event/EventPanel";
+import { EventSection } from "./event/EventSection";
+import { TranscriptEventState } from "./types";
+
+import styles from "./SampleInitEventView.module.css";
+
+interface SampleInitEventViewProps {
+  id: string;
+  event: SampleInitEvent;
+  eventState: TranscriptEventState;
+  setEventState: (state: TranscriptEventState) => void;
+  className?: string | string[];
+}
+
+/**
+ * Renders the SampleInitEventView component.
+ */
+export const SampleInitEventView: React.FC<SampleInitEventViewProps> = ({
+  id,
+  event,
+  eventState,
+  setEventState,
+  className,
+}) => {
+  const stateObj = event.state as Record<string, unknown>;
+
+  const sections = [];
+
+  if (event.sample.files && Object.keys(event.sample.files).length > 0) {
+    sections.push(
+      <EventSection title="Files" key={`sample-${id}-init-files`}>
+        {Object.keys(event.sample.files).map((file) => {
+          return (
+            <pre key={`sample-init-file-${file}`} className={styles.noMargin}>
+              {file}
+            </pre>
+          );
+        })}
+      </EventSection>,
+    );
+  }
+
+  if (event.sample.setup) {
+    sections.push(
+      <EventSection title="Setup" key={`sample-${id}-init-setup`}>
+        <pre className={styles.code}>
+          <code className="sourceCode">{event.sample.setup}</code>
+        </pre>
+      </EventSection>,
+    );
+  }
+
+  return (
+    <EventPanel
+      id={id}
+      className={className}
+      title="Sample"
+      icon={ApplicationIcons.sample}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      selectedNav={eventState.selectedNav || ""}
+      setSelectedNav={(selectedNav) => {
+        setEventState({ ...eventState, selectedNav });
+      }}
+      collapsed={eventState.collapsed}
+      setCollapsed={(collapsed) => {
+        setEventState({ ...eventState, collapsed });
+      }}
+    >
+      <div data-name="Sample" className={styles.sample}>
+        <ChatView messages={stateObj["messages"] as Messages} />
+        <div>
+          {event.sample.choices
+            ? event.sample.choices.map((choice, index) => {
+                return (
+                  <div key={`$choice-{choice}`}>
+                    {String.fromCharCode(65 + index)}) {choice}
+                  </div>
+                );
+              })
+            : ""}
+          {sections.length > 0 ? (
+            <div className={styles.section}>{sections}</div>
+          ) : (
+            ""
+          )}
+          <EventSection title="Target">
+            {toArray(event.sample.target).map((target) => {
+              return <div key={target}>{target}</div>;
+            })}
+          </EventSection>
+        </div>
+      </div>
+      {event.sample.metadata &&
+      Object.keys(event.sample.metadata).length > 0 ? (
+        <MetaDataGrid
+          data-name="Metadata"
+          className={styles.metadata}
+          entries={event.sample.metadata}
+        />
+      ) : (
+        ""
+      )}
+    </EventPanel>
+  );
+};

inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx

@@ -0,0 +1,75 @@
+import { ApplicationIcons } from "../../appearance/icons";
+import { SampleLimitEvent, Type7 } from "../../types/log";
+import { EventPanel } from "./event/EventPanel";
+import { TranscriptEventState } from "./types";
+
+interface SampleLimitEventViewProps {
+  id: string;
+  event: SampleLimitEvent;
+  eventState: TranscriptEventState;
+  setEventState: (state: TranscriptEventState) => void;
+  className?: string | string[];
+}
+
+/**
+ * Renders the InfoEventView component.
+ */
+export const SampleLimitEventView: React.FC<SampleLimitEventViewProps> = ({
+  id,
+  event,
+  eventState,
+  setEventState,
+  className,
+}) => {
+  const resolve_title = (type: Type7) => {
+    switch (type) {
+      case "custom":
+        return "Custom Limit Exceeded";
+      case "time":
+        return "Time Limit Execeeded";
+      case "message":
+        return "Message Limit Exceeded";
+      case "token":
+        return "Token Limit Exceeded";
+      case "operator":
+        return "Operator Canceled";
+    }
+  };
+
+  const resolve_icon = (type: Type7) => {
+    switch (type) {
+      case "custom":
+        return ApplicationIcons.limits.custom;
+      case "time":
+        return ApplicationIcons.limits.time;
+      case "message":
+        return ApplicationIcons.limits.messages;
+      case "token":
+        return ApplicationIcons.limits.tokens;
+      case "operator":
+        return ApplicationIcons.limits.operator;
+    }
+  };
+
+  const title = resolve_title(event.type);
+  const icon = resolve_icon(event.type);
+
+  return (
+    <EventPanel
+      id={id}
+      title={title}
+      icon={icon}
+      className={className}
+      selectedNav={eventState.selectedNav || ""}
+      setSelectedNav={(selectedNav) => {
+        setEventState({ ...eventState, selectedNav });
+      }}
+      collapsed={eventState.collapsed}
+      setCollapsed={(collapsed) => {
+        setEventState({ ...eventState, collapsed });
+      }}
+    >
+      {event.message}
+    </EventPanel>
+  );
+};

inspect_ai/_view/www/src/samples/transcript/SampleTranscript.tsx

@@ -0,0 +1,22 @@
+import { RefObject } from "react";
+import { Events } from "../../types/log";
+import { TranscriptVirtualList } from "./TranscriptView";
+
+interface SampleTranscriptProps {
+  id: string;
+  evalEvents: Events;
+  scrollRef: RefObject<HTMLDivElement | null>;
+}
+
+/**
+ * Renders the SampleTranscript component.=
+ */
+export const SampleTranscript: React.FC<SampleTranscriptProps> = ({
+  id,
+  evalEvents,
+  scrollRef,
+}) => {
+  return (
+    <TranscriptVirtualList id={id} events={evalEvents} scrollRef={scrollRef} />
+  );
+};

inspect_ai/_view/www/src/samples/transcript/ScoreEventView.module.css

@@ -0,0 +1,15 @@
+.explanation {
+  display: grid;
+  grid-template-columns: max-content auto;
+  column-gap: 1em;
+  margin: 0.5em 0;
+}
+
+.separator {
+  grid-column: 1 / -1;
+  border-bottom: solid 1px var(--bs-light-border-subtle);
+}
+
+.metadata {
+  margin: 0.5em 0;
+}

inspect_ai/_view/www/src/samples/transcript/ScoreEventView.tsx

@@ -0,0 +1,100 @@
+import { Fragment } from "react";
+import { ApplicationIcons } from "../../appearance/icons";
+import { MarkdownDiv } from "../../components/MarkdownDiv";
+import { MetaDataGrid } from "../../metadata/MetaDataGrid";
+import { ScoreEvent, Value1 } from "../../types/log";
+import { formatDateTime } from "../../utils/format";
+import { EventPanel } from "./event/EventPanel";
+import { TranscriptEventState } from "./types";
+
+import clsx from "clsx";
+import styles from "./ScoreEventView.module.css";
+
+interface ScoreEventViewProps {
+  id: string;
+  event: ScoreEvent;
+  eventState: TranscriptEventState;
+  setEventState: (state: TranscriptEventState) => void;
+  className?: string | string[];
+}
+
+/**
+ * Renders the ScoreEventView component.
+ */
+export const ScoreEventView: React.FC<ScoreEventViewProps> = ({
+  id,
+  event,
+  eventState,
+  setEventState,
+  className,
+}) => {
+  const resolvedTarget = event.target
+    ? Array.isArray(event.target)
+      ? event.target.join("\n")
+      : event.target
+    : undefined;
+
+  return (
+    <EventPanel
+      id={id}
+      title={(event.intermediate ? "Intermediate " : "") + "Score"}
+      className={clsx(className, "text-size-small")}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      icon={ApplicationIcons.scorer}
+      selectedNav={eventState.selectedNav || ""}
+      setSelectedNav={(selectedNav) => {
+        setEventState({ ...eventState, selectedNav });
+      }}
+      collapsed={eventState.collapsed}
+      setCollapsed={(collapsed) => {
+        setEventState({ ...eventState, collapsed });
+      }}
+    >
+      <div data-name="Explanation" className={clsx(styles.explanation)}>
+        {event.target ? (
+          <Fragment>
+            <div className={clsx(styles.separator)}></div>
+            <div className={"text-style-label"}>Target</div>
+            <div>
+              <MarkdownDiv markdown={resolvedTarget || ""} />
+            </div>
+          </Fragment>
+        ) : (
+          ""
+        )}
+        <div className={clsx(styles.separator)}></div>
+        <div className={"text-style-label"}>Answer</div>
+        <div>
+          <MarkdownDiv markdown={event.score.answer || ""} />
+        </div>
+        <div className={clsx(styles.separator)}></div>
+        <div className={"text-style-label"}>Explanation</div>
+        <div>
+          <MarkdownDiv markdown={event.score.explanation || ""} />
+        </div>
+        <div className={clsx(styles.separator)}></div>
+        <div className={"text-style-label"}>Score</div>
+        <div>{renderScore(event.score.value)}</div>
+        <div className={clsx(styles.separator)}></div>
+      </div>
+      {event.score.metadata ? (
+        <div data-name="Metadata">
+          <MetaDataGrid
+            entries={event.score.metadata}
+            className={styles.metadata}
+          />
+        </div>
+      ) : undefined}
+    </EventPanel>
+  );
+};
+
+const renderScore = (value: Value1) => {
+  if (Array.isArray(value)) {
+    return value.join(" ");
+  } else if (typeof value === "object") {
+    return <MetaDataGrid entries={value} />;
+  } else {
+    return value;
+  }
+};

inspect_ai/_view/www/src/samples/transcript/StepEventView.tsx

@@ -0,0 +1,171 @@
+import clsx from "clsx";
+import { RefObject, useCallback, useState } from "react";
+import { StepEvent } from "../../types/log";
+import { formatDateTime } from "../../utils/format";
+import { EventPanel } from "./event/EventPanel";
+import { TranscriptVirtualListComponent } from "./TranscriptView";
+import { EventNode, TranscriptEventState } from "./types";
+
+interface StepEventViewProps {
+  event: StepEvent;
+  eventState: TranscriptEventState;
+  setEventState: (state: TranscriptEventState) => void;
+  children: EventNode[];
+  scrollRef?: RefObject<HTMLDivElement | null>;
+  className?: string | string[];
+}
+
+/**
+ * Renders the StepEventView component.
+ */
+export const StepEventView: React.FC<StepEventViewProps> = ({
+  event,
+  eventState,
+  setEventState,
+  children,
+  scrollRef,
+  className,
+}) => {
+  const descriptor = stepDescriptor(event);
+  const title =
+    descriptor.name ||
+    `${event.type ? event.type + ": " : "Step: "}${event.name}`;
+  const text = summarize(children);
+
+  const [transcriptState, setTranscriptState] = useState({});
+  const onTranscriptState = useCallback(
+    (state: TranscriptEventState) => {
+      setTranscriptState({ ...state });
+    },
+    [transcriptState, setTranscriptState],
+  );
+
+  return (
+    <EventPanel
+      id={`step-${event.name}`}
+      className={clsx("transcript-step", className)}
+      title={title}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      icon={descriptor.icon}
+      collapse={false}
+      text={text}
+      selectedNav={eventState.selectedNav || ""}
+      setSelectedNav={(selectedNav) => {
+        setEventState({ ...eventState, selectedNav });
+      }}
+      collapsed={eventState.collapsed}
+      setCollapsed={(collapsed) => {
+        setEventState({ ...eventState, collapsed });
+      }}
+    >
+      <TranscriptVirtualListComponent
+        id={`step-${event.name}-transcript`}
+        eventNodes={children}
+        scrollRef={scrollRef}
+        transcriptState={transcriptState}
+        setTranscriptState={onTranscriptState}
+      />
+    </EventPanel>
+  );
+};
+
+/**
+ * Renders the StepEventView component.
+ */
+const summarize = (children: EventNode[]) => {
+  if (children.length === 0) {
+    return "(no events)";
+  }
+
+  const formatEvent = (event: string, count: number) => {
+    if (count === 1) {
+      return `${count} ${event} event`;
+    } else {
+      return `${count} ${event} events`;
+    }
+  };
+
+  // Count the types
+  const typeCount: Record<string, number> = {};
+  children.forEach((child) => {
+    const currentCount = typeCount[child.event.event] || 0;
+    typeCount[child.event.event] = currentCount + 1;
+  });
+
+  // Try to summarize event types
+  const numberOfTypes = Object.keys(typeCount).length;
+  if (numberOfTypes < 3) {
+    return Object.keys(typeCount)
+      .map((key) => {
+        return formatEvent(key, typeCount[key]);
+      })
+      .join(", ");
+  }
+
+  // To many types, just return the number of events
+  if (children.length === 1) {
+    return "1 event";
+  } else {
+    return `${children.length} events`;
+  }
+};
+
+/**
+ * Returns a descriptor object containing icon and style based on the event type and name.
+ */
+const stepDescriptor = (
+  event: StepEvent,
+): { icon?: string; name?: string; endSpace?: boolean } => {
+  const rootStepDescriptor = {
+    endSpace: true,
+  };
+
+  if (event.type === "solver") {
+    switch (event.name) {
+      case "chain_of_thought":
+        return {
+          ...rootStepDescriptor,
+        };
+      case "generate":
+        return {
+          ...rootStepDescriptor,
+        };
+      case "self_critique":
+        return {
+          ...rootStepDescriptor,
+        };
+      case "system_message":
+        return {
+          ...rootStepDescriptor,
+        };
+      case "use_tools":
+        return {
+          ...rootStepDescriptor,
+        };
+      case "multiple_choice":
+        return {
+          ...rootStepDescriptor,
+        };
+      default:
+        return {
+          ...rootStepDescriptor,
+        };
+    }
+  } else if (event.type === "scorer") {
+    return {
+      ...rootStepDescriptor,
+    };
+  } else {
+    switch (event.name) {
+      case "sample_init":
+        return {
+          ...rootStepDescriptor,
+          name: "Sample Init",
+        };
+      default:
+        return {
+          endSpace: false,
+        };
+    }
+  }
+};

inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.module.css

@@ -0,0 +1,19 @@
+.summary {
+  width: 100%;
+  margin: 0.5em 0em;
+}
+
+.summaryRendered {
+  margin-bottom: 1em;
+}
+
+.subtaskSummary {
+  display: grid;
+  grid-template-columns: minmax(0, 1fr) max-content minmax(0, 1fr);
+  column-gap: 1em;
+  margin: 0.5em 0;
+}
+
+.subtaskLabel {
+  padding: 0 2em;
+}

inspect_ai/_view/www/src/samples/transcript/SubtaskEventView.tsx

@@ -0,0 +1,133 @@
+import clsx from "clsx";
+import { Fragment } from "react";
+import { ApplicationIcons } from "../../appearance/icons";
+import { MetaDataView } from "../../metadata/MetaDataView";
+import { Input2, Input4, Result1, SubtaskEvent } from "../../types/log";
+import { formatDateTime } from "../../utils/format";
+import { EventPanel } from "./event/EventPanel";
+import styles from "./SubtaskEventView.module.css";
+import { TranscriptView } from "./TranscriptView";
+import { TranscriptEventState } from "./types";
+
+interface SubtaskEventViewProps {
+  id: string;
+  event: SubtaskEvent;
+  eventState: TranscriptEventState;
+  setEventState: (state: TranscriptEventState) => void;
+  depth: number;
+  className?: string | string[];
+}
+
+/**
+ * Renders the StateEventView component.
+ */
+export const SubtaskEventView: React.FC<SubtaskEventViewProps> = ({
+  id,
+  event,
+  eventState,
+  setEventState,
+  depth,
+  className,
+}) => {
+  // Render Forks specially
+
+  const transcript =
+    event.events.length > 0 ? (
+      <TranscriptView
+        id={`${id}-subtask`}
+        data-name="Transcript"
+        events={event.events}
+        depth={depth + 1}
+      />
+    ) : (
+      ""
+    );
+
+  const body =
+    event.type === "fork" ? (
+      <div title="Summary" className={clsx(styles.summary)}>
+        <div className={clsx("text-style-label")}>Inputs</div>
+        <div className={clsx(styles.summaryRendered)}>
+          <Rendered values={event.input} />
+        </div>
+        <div className={clsx("text-style-label")}>Transcript</div>
+        {transcript}
+      </div>
+    ) : (
+      <Fragment>
+        <SubtaskSummary
+          data-name="Summary"
+          input={event.input}
+          result={event.result}
+        />
+        {transcript}
+      </Fragment>
+    );
+
+  // Is this a traditional subtask or a fork?
+  const type = event.type === "fork" ? "Fork" : "Subtask";
+  return (
+    <EventPanel
+      id={id}
+      className={className}
+      title={`${type}: ${event.name}`}
+      subTitle={formatDateTime(new Date(event.timestamp))}
+      collapse={false}
+      selectedNav={eventState.selectedNav || ""}
+      setSelectedNav={(selectedNav) => {
+        setEventState({ ...eventState, selectedNav });
+      }}
+      collapsed={eventState.collapsed}
+      setCollapsed={(collapsed) => {
+        setEventState({ ...eventState, collapsed });
+      }}
+    >
+      {body}
+    </EventPanel>
+  );
+};
+
+interface SubtaskSummaryProps {
+  input: Input2 | Input4;
+  result: Result1;
+}
+/**
+ * Renders the StateEventView component.
+ */
+const SubtaskSummary: React.FC<SubtaskSummaryProps> = ({ input, result }) => {
+  result = typeof result === "object" ? result : { result };
+  return (
+    <div className={clsx(styles.subtaskSummary)}>
+      <div className={clsx("text-style-label")}>Input</div>
+      <div className={clsx("text-size-large", styles.subtaskLabel)}></div>
+      <div className={clsx("text-style-label")}>Output</div>
+      <Rendered values={input} />
+      <div className={clsx("text-size-title-secondary", styles.subtaskLabel)}>
+        <i className={ApplicationIcons.arrows.right} />
+      </div>
+      <div>
+        <Rendered values={result} />
+      </div>
+    </div>
+  );
+};
+
+interface RenderedProps {
+  values: Array<unknown> | Object | string | number;
+}
+
+/**
+ * Recursively renders content based on the type of `values`.
+value.
+ */
+const Rendered: React.FC<RenderedProps> = ({ values }) => {
+  if (Array.isArray(values)) {
+    return values.map((val) => {
+      return <Rendered values={val} />;
+    });
+  } else if (values && typeof values === "object") {
+    return <MetaDataView entries={values as Record<string, unknown>} />;
+  } else {
+    return values;
+  }
+};

inspect_ai/_view/www/src/samples/transcript/ToolEventView.module.css

@@ -0,0 +1,10 @@
+.summary {
+  margin: 0.5em 0;
+  width: 100%;
+}
+
+.approval {
+  border: none;
+  padding: 0;
+  margin-bottom: 0;
+}