inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/samples/SampleDisplay.tsx

@@ -0,0 +1,331 @@
+import { TabPanel, TabSet } from "../components/TabSet";
+import { MetaDataView } from "../metadata/MetaDataView";
+
+import { escapeSelector } from "../utils/html";
+import { isVscode } from "../utils/vscode";
+
+import { ApplicationIcons } from "../appearance/icons";
+import { ANSIDisplay } from "../components/AnsiDisplay";
+import { ToolButton } from "../components/ToolButton";
+import { SampleScoreView } from "./scores/SampleScoreView";
+
+import clsx from "clsx";
+import { Fragment, MouseEvent, RefObject } from "react";
+import { Card, CardBody, CardHeader } from "../components/Card";
+import { EmptyPanel } from "../components/EmptyPanel";
+import { JSONPanel } from "../components/JsonPanel";
+import {
+  kSampleErrorTabId,
+  kSampleJsonTabId,
+  kSampleMessagesTabId,
+  kSampleMetdataTabId,
+  kSampleScoringTabId,
+  kSampleTranscriptTabId,
+} from "../constants";
+import { EvalSample } from "../types/log";
+import { ModelTokenTable } from "../usage/ModelTokenTable";
+import { printHeadingHtml, printHtml } from "../utils/print";
+import { ChatViewVirtualList } from "./chat/ChatViewVirtualList";
+import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
+import styles from "./SampleDisplay.module.css";
+import { SampleSummaryView } from "./SampleSummaryView";
+import { SampleTranscript } from "./transcript/SampleTranscript";
+
+interface SampleDisplayProps {
+  id: string;
+  sample?: EvalSample;
+  sampleDescriptor: SamplesDescriptor;
+  selectedTab?: string;
+  setSelectedTab: (tab: string) => void;
+  scrollRef: RefObject<HTMLDivElement | null>;
+}
+
+/**
+ * Component to display a sample with relevant context and visibility control.
+ */
+export const SampleDisplay: React.FC<SampleDisplayProps> = ({
+  id,
+  sample,
+  sampleDescriptor,
+  selectedTab,
+  setSelectedTab,
+  scrollRef,
+}) => {
+  // Tab ids
+  const baseId = `sample-dialog`;
+
+  if (!sample) {
+    // Placeholder
+    return <EmptyPanel />;
+  }
+
+  // Tab selection
+  const onSelectedTab = (e: MouseEvent<HTMLElement>) => {
+    const el = e.currentTarget as HTMLElement;
+    const id = el.id;
+    setSelectedTab(id);
+    return false;
+  };
+
+  const scorerNames = Object.keys(sample.scores || {});
+  const sampleMetadatas = metadataViewsForSample(`${baseId}-${id}`, sample);
+
+  const tabsetId = `task-sample-details-tab-${id}`;
+  const targetId = `${tabsetId}-content`;
+
+  const tools = [];
+  if (!isVscode()) {
+    tools.push(
+      <ToolButton
+        key="sample-print-tool"
+        label="Print"
+        icon={ApplicationIcons.copy}
+        onClick={() => {
+          printSample(id, targetId);
+        }}
+      />,
+    );
+  }
+
+  return (
+    <Fragment>
+      <SampleSummaryView
+        parent_id={id}
+        sample={sample}
+        sampleDescriptor={sampleDescriptor}
+      />
+      <TabSet
+        id={tabsetId}
+        tabControlsClassName={clsx("text-size-base")}
+        tabPanelsClassName={clsx(styles.tabPanel)}
+        tools={tools}
+      >
+        {sample.events && sample.events.length > 0 ? (
+          <TabPanel
+            key={kSampleTranscriptTabId}
+            id={kSampleTranscriptTabId}
+            className="sample-tab"
+            title="Transcript"
+            onSelected={onSelectedTab}
+            selected={
+              selectedTab === kSampleTranscriptTabId ||
+              selectedTab === undefined
+            }
+            scrollable={false}
+          >
+            <SampleTranscript
+              key={`${baseId}-transcript-display-${id}`}
+              id={`${baseId}-transcript-display-${id}`}
+              evalEvents={sample.events}
+              scrollRef={scrollRef}
+            />
+          </TabPanel>
+        ) : null}
+        <TabPanel
+          key={kSampleMessagesTabId}
+          id={kSampleMessagesTabId}
+          className={clsx("sample-tab", styles.fullWidth)}
+          title="Messages"
+          onSelected={onSelectedTab}
+          selected={selectedTab === kSampleMessagesTabId}
+          scrollable={false}
+        >
+          <ChatViewVirtualList
+            key={`${baseId}-chat-${id}`}
+            id={`${baseId}-chat-${id}`}
+            messages={sample.messages}
+            indented={true}
+            scrollRef={scrollRef}
+            toolCallStyle="complete"
+          />
+        </TabPanel>
+        {scorerNames.length === 1 ? (
+          <TabPanel
+            key={kSampleScoringTabId}
+            id={kSampleScoringTabId}
+            className="sample-tab"
+            title="Scoring"
+            onSelected={onSelectedTab}
+            selected={selectedTab === kSampleScoringTabId}
+          >
+            <SampleScoreView
+              sample={sample}
+              sampleDescriptor={sampleDescriptor}
+              scorer={scorerNames[0]}
+            />
+          </TabPanel>
+        ) : (
+          <>
+            {Object.keys(sample.scores || {}).map((scorer) => {
+              const tabId = `score-${scorer}`;
+              return (
+                <TabPanel
+                  key={tabId}
+                  id={tabId}
+                  className="sample-tab"
+                  title={scorer}
+                  onSelected={onSelectedTab}
+                  selected={selectedTab === tabId}
+                >
+                  <SampleScoreView
+                    sample={sample}
+                    sampleDescriptor={sampleDescriptor}
+                    scorer={scorer}
+                  />
+                </TabPanel>
+              );
+            })}
+          </>
+        )}
+        {sampleMetadatas.length > 0 ? (
+          <TabPanel
+            id={kSampleMetdataTabId}
+            className="sample-tab"
+            title="Metadata"
+            onSelected={onSelectedTab}
+            selected={selectedTab === kSampleMetdataTabId}
+          >
+            <div className={clsx(styles.metadataPanel)}>{sampleMetadatas}</div>
+          </TabPanel>
+        ) : null}
+        {sample.error ? (
+          <TabPanel
+            id={kSampleErrorTabId}
+            className="sample-tab"
+            title="Error"
+            onSelected={onSelectedTab}
+            selected={selectedTab === kSampleErrorTabId}
+          >
+            <div className={clsx(styles.padded)}>
+              <ANSIDisplay
+                output={sample.error.traceback_ansi}
+                className={clsx("text-size-small", styles.ansi)}
+              />
+            </div>
+          </TabPanel>
+        ) : null}
+        {sample.messages.length < 100 ? (
+          <TabPanel
+            id={kSampleJsonTabId}
+            className={"sample-tab"}
+            title="JSON"
+            onSelected={onSelectedTab}
+            selected={selectedTab === kSampleJsonTabId}
+          >
+            <div className={clsx(styles.padded, styles.fullWidth)}>
+              <JSONPanel data={sample} simple={true} />
+            </div>
+          </TabPanel>
+        ) : null}
+      </TabSet>
+    </Fragment>
+  );
+};
+
+const metadataViewsForSample = (id: string, sample: EvalSample) => {
+  const sampleMetadatas = [];
+  if (sample.model_usage && Object.keys(sample.model_usage).length > 0) {
+    sampleMetadatas.push(
+      <Card key={`sample-usage-${id}`}>
+        <CardHeader label="Usage" />
+        <CardBody>
+          <ModelTokenTable
+            model_usage={sample.model_usage}
+            className={clsx(styles.noTop)}
+          />
+        </CardBody>
+      </Card>,
+    );
+  }
+
+  if (Object.keys(sample?.metadata).length > 0) {
+    sampleMetadatas.push(
+      <Card key={`sample-metadata-${id}`}>
+        <CardHeader label="Metadata" />
+        <CardBody>
+          <MetaDataView
+            id="task-sample-metadata-${id}"
+            entries={sample?.metadata as Record<string, unknown>}
+            className={clsx("tab-pane", styles.noTop)}
+          />
+        </CardBody>
+      </Card>,
+    );
+  }
+
+  if (Object.keys(sample?.store).length > 0) {
+    sampleMetadatas.push(
+      <Card key={`sample-store-${id}`}>
+        <CardHeader label="Store" />
+        <CardBody>
+          <MetaDataView
+            id="task-sample-store-${id}"
+            entries={sample?.store as Record<string, unknown>}
+            className={clsx("tab-pane", styles.noTop)}
+          />
+        </CardBody>
+      </Card>,
+    );
+  }
+
+  return sampleMetadatas;
+};
+const printSample = (id: string, targetId: string) => {
+  // The active tab
+  const targetTabEl = document.querySelector(
+    `#${escapeSelector(targetId)} .sample-tab.tab-pane.show.active`,
+  );
+  if (targetTabEl) {
+    // The target element
+    const targetEl = targetTabEl.firstElementChild;
+    if (targetEl) {
+      // Get the sample heading to include
+      const headingId = `sample-heading-${id}`;
+      const headingEl = document.getElementById(headingId);
+
+      // Print the document
+      const headingHtml = printHeadingHtml();
+      const css = `
+      html { font-size: 9pt }
+      /* Allow content to break anywhere without any forced page breaks */
+      * {
+        break-inside: auto;  /* Let elements break anywhere */
+        page-break-inside: auto;  /* Legacy support */
+        break-before: auto;
+        page-break-before: auto;
+        break-after: auto;
+        page-break-after: auto;
+      }
+      /* Specifically disable all page breaks for divs */
+      div {
+        break-inside: auto;
+        page-break-inside: auto;
+      }
+      body > .transcript-step {
+        break-inside: avoid;
+      }
+      body{
+        -webkit-print-color-adjust:exact !important;
+        print-color-adjust:exact !important;
+      }
+      /* Allow preformatted text and code blocks to break across pages */
+      pre, code {
+          white-space: pre-wrap; /* Wrap long lines instead of keeping them on one line */
+          overflow-wrap: break-word; /* Ensure long words are broken to fit within the page */
+          break-inside: auto; /* Allow page breaks inside the element */
+          page-break-inside: auto; /* Older equivalent */
+      }
+
+      /* Additional control for long lines within code/preformatted blocks */
+      pre {
+          word-wrap: break-word; /* Break long words if needed */
+      }    
+          
+      `;
+      printHtml(
+        [headingHtml, headingEl?.outerHTML, targetEl.innerHTML].join("\n"),
+        css,
+      );
+    }
+  }
+};

inspect_ai/_view/www/src/samples/SampleSummaryView.module.css

@@ -0,0 +1,24 @@
+.target {
+  padding-left: 0;
+}
+
+.answer {
+  padding-left: 0;
+}
+
+.grid {
+  display: grid;
+  grid-column-gap: 0.5em;
+  border-bottom: solid var(--bs-border-color) 1px;
+  margin-bottom: 1em;
+  padding: 0em 1em 1em 1em;
+}
+
+.centerLabel {
+  display: flex;
+  justify-content: center;
+}
+
+.wrap {
+  word-wrap: anywhere;
+}

inspect_ai/_view/www/src/samples/SampleSummaryView.tsx

@@ -0,0 +1,177 @@
+import clsx from "clsx";
+import { MarkdownDiv } from "../components/MarkdownDiv";
+import { EvalSample } from "../types/log";
+import { arrayToString, inputString } from "../utils/format";
+import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
+import { FlatSampleError } from "./error/FlatSampleErrorView";
+
+import { ReactNode } from "react";
+import styles from "./SampleSummaryView.module.css";
+
+interface SampleSummaryViewProps {
+  parent_id: string;
+  sample: EvalSample;
+  sampleDescriptor: SamplesDescriptor;
+}
+
+interface SummaryColumn {
+  label: string;
+  value: string | ReactNode;
+  size: string;
+  center?: boolean;
+  clamp?: boolean;
+}
+
+/**
+ * Component to display a sample with relevant context and visibility control.
+ */
+export const SampleSummaryView: React.FC<SampleSummaryViewProps> = ({
+  parent_id,
+  sample,
+  sampleDescriptor,
+}) => {
+  const input =
+    sampleDescriptor?.messageShape.normalized.input > 0
+      ? Math.max(0.15, sampleDescriptor.messageShape.normalized.input)
+      : 0;
+  const target =
+    sampleDescriptor?.messageShape.normalized.target > 0
+      ? Math.max(0.15, sampleDescriptor.messageShape.normalized.target)
+      : 0;
+  const answer =
+    sampleDescriptor?.messageShape.normalized.answer > 0
+      ? Math.max(0.15, sampleDescriptor.messageShape.normalized.answer)
+      : 0;
+  const limitSize =
+    sampleDescriptor?.messageShape.normalized.limit > 0
+      ? Math.max(0.15, sampleDescriptor.messageShape.normalized.limit)
+      : 0;
+  const idSize = Math.max(
+    2,
+    Math.min(10, sampleDescriptor?.messageShape.raw.id),
+  );
+
+  const scoreInput = inputString(sample.input);
+  if (sample.choices && sample.choices.length > 0) {
+    scoreInput.push("");
+    scoreInput.push(
+      ...sample.choices.map((choice, index) => {
+        return `${String.fromCharCode(65 + index)}) ${choice}`;
+      }),
+    );
+  }
+
+  // The columns for the sample
+  const columns: SummaryColumn[] = [];
+  columns.push({
+    label: "Id",
+    value: sample.id,
+    size: `${idSize}em`,
+  });
+
+  columns.push({
+    label: "Input",
+    value: scoreInput,
+    size: `${input}fr`,
+    clamp: true,
+  });
+
+  if (sample.target) {
+    columns.push({
+      label: "Target",
+      value: (
+        <MarkdownDiv
+          markdown={arrayToString(arrayToString(sample?.target || "none"))}
+          className={clsx("no-last-para-padding", styles.target)}
+        />
+      ),
+      size: `${target}fr`,
+      clamp: true,
+    });
+  }
+
+  const fullAnswer =
+    sample && sampleDescriptor
+      ? sampleDescriptor.selectedScorerDescriptor(sample).answer()
+      : undefined;
+  if (fullAnswer) {
+    columns.push({
+      label: "Answer",
+      value: sample ? (
+        <MarkdownDiv
+          markdown={fullAnswer}
+          className={clsx("no-last-para-padding", styles.answer)}
+        />
+      ) : (
+        ""
+      ),
+      size: `${answer}fr`,
+      clamp: true,
+    });
+  }
+
+  if (sample?.limit && limitSize > 0) {
+    columns.push({
+      label: "Limit",
+      value: sample.limit.type,
+      size: `${limitSize}fr`,
+      center: true,
+    });
+  }
+
+  columns.push({
+    label: "Score",
+    value: sample.error ? (
+      <FlatSampleError message={sample.error.message} />
+    ) : (
+      // TODO: Cleanup once the PR lands which makes sample / sample summary share common interface
+      sampleDescriptor?.selectedScore(sample)?.render() || ""
+    ),
+    size: "minmax(2em, auto)",
+    center: true,
+  });
+
+  return (
+    <div
+      id={`sample-heading-${parent_id}`}
+      className={clsx(styles.grid, "text-size-base")}
+      style={{
+        gridTemplateColumns: `${columns
+          .map((col) => {
+            return col.size;
+          })
+          .join(" ")}`,
+      }}
+    >
+      {columns.map((col, idx) => {
+        return (
+          <div
+            key={`sample-summ-lbl-${idx}`}
+            className={clsx(
+              "text-style-label",
+              "text-style-secondary",
+              "text-size-base",
+              col.center ? styles.centerLabel : undefined,
+            )}
+          >
+            {col.label}
+          </div>
+        );
+      })}
+      {columns.map((col, idx) => {
+        return (
+          <div
+            key={`sample-summ-val-${idx}`}
+            className={clsx(
+              styles.wrap,
+              col.clamp ? "three-line-clamp" : undefined,
+              col.center ? styles.centerLabel : undefined,
+            )}
+          >
+            {col.value}
+          </div>
+        );
+      })}
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/SamplesTools.tsx

@@ -0,0 +1,52 @@
+import { Fragment } from "react/jsx-runtime";
+import { ScoreFilter, ScoreLabel } from "../types";
+import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
+import { EpochFilter } from "./sample-tools/EpochFilter";
+import { SampleFilter } from "./sample-tools/sample-filter/SampleFilter";
+import { SelectScorer } from "./sample-tools/SelectScorer";
+import { SortFilter } from "./sample-tools/SortFilter";
+
+interface SampleToolsProps {
+  epoch: string;
+  setEpoch: (epoch: string) => void;
+  epochs: number;
+  scoreFilter: ScoreFilter;
+  setScoreFilter: (filter: ScoreFilter) => void;
+  sort: string;
+  setSort: (sort: string) => void;
+  score?: ScoreLabel;
+  setScore: (score: ScoreLabel) => void;
+  scores: ScoreLabel[];
+  sampleDescriptor: SamplesDescriptor;
+}
+
+export const SampleTools: React.FC<SampleToolsProps> = ({
+  epoch,
+  setEpoch,
+  epochs,
+  scoreFilter,
+  setScoreFilter,
+  sort,
+  setSort,
+  score,
+  setScore,
+  scores,
+  sampleDescriptor,
+}) => {
+  return (
+    <Fragment>
+      <SampleFilter
+        evalDescriptor={sampleDescriptor.evalDescriptor}
+        scoreFilter={scoreFilter}
+        setScoreFilter={setScoreFilter}
+      />
+      {scores.length > 1 ? (
+        <SelectScorer scores={scores} score={score} setScore={setScore} />
+      ) : undefined}
+      {epochs > 1 ? (
+        <EpochFilter epoch={epoch} setEpoch={setEpoch} epochs={epochs} />
+      ) : undefined}
+      <SortFilter sort={sort} setSort={setSort} epochs={epochs} />
+    </Fragment>
+  );
+};

inspect_ai/_view/www/src/samples/chat/ChatMessage.module.css

@@ -0,0 +1,29 @@
+.message {
+  font-weight: 300;
+  padding-bottom: 0.5em;
+  margin-left: 0;
+  margin-right: 0;
+  white-space: normal;
+}
+
+.systemRole {
+  opacity: 0.7;
+}
+
+.messageGrid {
+  display: grid;
+  grid-template-columns: max-content auto;
+  column-gap: 0.3em;
+  font-weight: 500;
+  margin-bottom: 0.5em;
+}
+
+.messageContents {
+  margin-left: 0;
+  padding-bottom: 0;
+}
+
+.messageContents.indented {
+  margin-left: 1.1rem;
+  padding-bottom: 0.8rem;
+}

inspect_ai/_view/www/src/samples/chat/ChatMessage.tsx

@@ -0,0 +1,76 @@
+import clsx from "clsx";
+import { Fragment } from "react";
+import ExpandablePanel from "../../components/ExpandablePanel";
+import { MarkdownDiv } from "../../components/MarkdownDiv";
+import {
+  ChatMessageAssistant,
+  ChatMessageSystem,
+  ChatMessageTool,
+  ChatMessageUser,
+} from "../../types/log";
+import styles from "./ChatMessage.module.css";
+import { MessageContents } from "./MessageContents";
+import { iconForMsg } from "./messages";
+
+interface ChatMessageProps {
+  id: string;
+  message: ChatMessageAssistant | ChatMessageSystem | ChatMessageUser;
+  toolMessages: ChatMessageTool[];
+  indented?: boolean;
+  toolCallStyle: "compact" | "complete";
+}
+
+export const ChatMessage: React.FC<ChatMessageProps> = ({
+  id,
+  message,
+  toolMessages,
+  indented,
+  toolCallStyle,
+}) => {
+  const collapse = message.role === "system";
+  return (
+    <div
+      className={clsx(
+        message.role,
+        "text-size-base",
+        styles.message,
+        message.role === "system" ? styles.systemRole : undefined,
+      )}
+    >
+      <div className={clsx(styles.messageGrid, "text-style-label")}>
+        <i className={iconForMsg(message)} />
+        {message.role}
+      </div>
+      {message.role === "assistant" && message.reasoning ? (
+        <Fragment key={`${id}-response-label`}>
+          <div className={clsx("text-style-label", "text-style-secondary")}>
+            Reasoning
+          </div>
+          <ExpandablePanel collapse={true}>
+            <MarkdownDiv markdown={message.reasoning} />
+          </ExpandablePanel>
+        </Fragment>
+      ) : undefined}
+      <div
+        className={clsx(
+          styles.messageContents,
+          indented ? styles.indented : undefined,
+        )}
+      >
+        {message.role === "assistant" && message.reasoning ? (
+          <div className={clsx("text-style-label", "text-style-secondary")}>
+            Response
+          </div>
+        ) : undefined}
+        <ExpandablePanel collapse={collapse}>
+          <MessageContents
+            key={`${id}-contents`}
+            message={message}
+            toolMessages={toolMessages}
+            toolCallStyle={toolCallStyle}
+          />
+        </ExpandablePanel>
+      </div>
+    </div>
+  );
+};