inspect-ai 0.3.62__py3-none-any.whl → 0.3.63__py3-none-any.whl

inspect_ai/_view/www/src/workspace/sidebar/SidebarLogEntry.tsx

@@ -0,0 +1,95 @@
+import clsx from "clsx";
+import { Fragment } from "react";
+import { EvalLogHeader } from "../../api/types";
+import { EvalStatus } from "./EvalStatus";
+import styles from "./SidebarLogEntry.module.css";
+
+interface SidebarLogEntryProps {
+  logHeader?: EvalLogHeader;
+  task: string;
+}
+
+export const SidebarLogEntry: React.FC<SidebarLogEntryProps> = ({
+  logHeader,
+  task,
+}) => {
+  const hyperparameters: Record<string, unknown> = {
+    ...(logHeader?.plan?.config || {}),
+    ...(logHeader?.eval?.task_args || {}),
+  };
+
+  const model = logHeader?.eval?.model;
+  const datasetName = logHeader?.eval?.dataset.name;
+
+  const uniqScorers = new Set();
+  logHeader?.results?.scores?.forEach((scorer) => {
+    uniqScorers.add(scorer.name);
+  });
+  const scorerNames = Array.from(uniqScorers).join(",");
+
+  const scorerLabel =
+    Object.keys(logHeader?.results?.scores || {}).length === 1
+      ? "scorer"
+      : "scorers";
+
+  const completed = logHeader?.stats?.completed_at;
+  const time = completed ? new Date(completed) : undefined;
+  const timeStr = time
+    ? `${time.toDateString()}
+    ${time.toLocaleTimeString([], {
+      hour: "2-digit",
+      minute: "2-digit",
+    })}`
+    : "";
+
+  return (
+    <Fragment>
+      <div className={styles.entry}>
+        <div className={styles.title}>
+          <div className={clsx(styles.task, "text-size-title-secondary")}>
+            {logHeader?.eval?.task || task}
+          </div>
+          <small className={clsx("mb-1", "text-size-small")}>{timeStr}</small>
+
+          {model ? (
+            <div>
+              <small className={clsx("mb-1", "text-size-small")}>{model}</small>
+            </div>
+          ) : (
+            ""
+          )}
+        </div>
+        <EvalStatus logHeader={logHeader} />
+      </div>
+      <div className={clsx(styles.params, "three-line-clamp")}>
+        <small className={"mb-1"}>
+          {hyperparameters
+            ? Object.keys(hyperparameters)
+                .map((key) => {
+                  const val = hyperparameters[key];
+                  if (Array.isArray(val) || typeof val === "object") {
+                    return `${key}: ${JSON.stringify(val)}`;
+                  } else {
+                    return `${key}: ${val}`;
+                  }
+                })
+                .join(", ")
+            : ""}
+        </small>
+      </div>
+      {(logHeader?.eval?.dataset || logHeader?.results?.scores) &&
+      logHeader?.status === "success" ? (
+        <div
+          className={clsx("text-truncate", "text-size-small", styles.scores)}
+        >
+          <div>dataset: {datasetName || "(samples)"}</div>
+          <div className={clsx("text-truncate", styles.scoreInfo)}>
+            {scorerLabel}: {scorerNames || "(none)"}
+          </div>
+        </div>
+      ) : (
+        ""
+      )}
+    </Fragment>
+  );
+};

inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.module.css

@@ -0,0 +1,23 @@
+.container {
+  display: flex;
+  flex-direction: row;
+  flex-wrap: wrap;
+  justify-content: flex-end;
+}
+
+.metric {
+  display: flex;
+  flex-direction: column;
+  align-items: flex-end;
+  margin-left: 1em;
+  margin-bottom: 0.4em;
+  margin-top: 0.5rem;
+}
+
+.metricName {
+  margin-bottom: -0.3em;
+}
+
+.metricReducer {
+  margin-bottom: -0.2rem;
+}

inspect_ai/_view/www/src/workspace/sidebar/SidebarScoreView.tsx

@@ -0,0 +1,41 @@
+import clsx from "clsx";
+import { EvalScore } from "../../types/log";
+import { formatPrettyDecimal } from "../../utils/format";
+
+import styles from "./SidebarScoreView.module.css";
+interface SidebarScoreProps {
+  scorer: EvalScore;
+}
+
+export const SidebarScoreView: React.FC<SidebarScoreProps> = ({ scorer }) => {
+  return (
+    <div className={styles.container}>
+      {Object.keys(scorer.metrics).map((metric) => {
+        return (
+          <div className={styles.metric}>
+            <div
+              className={clsx(
+                "text-style-secondary",
+                "text-style-label",
+                "text-size-small",
+                styles.metricName,
+              )}
+            >
+              {scorer.metrics[metric].name}
+            </div>
+            {scorer.reducer ? (
+              <div className={clsx("text-size-small", styles.metricReducer)}>
+                ${scorer.reducer}
+              </div>
+            ) : (
+              ""
+            )}
+            <div className={"text-size-title-secondary"}>
+              {formatPrettyDecimal(scorer.metrics[metric].value)}
+            </div>
+          </div>
+        );
+      })}
+    </div>
+  );
+};

inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.module.css

@@ -0,0 +1,35 @@
+.container {
+  display: flex;
+  flex-direction: row;
+  flex-wrap: wrap;
+  justify-content: flex-end;
+  row-gap: 1em;
+}
+
+.scoreWrapper {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  margin-left: 1em;
+}
+
+.metricName {
+  width: 100%;
+  font-weight: 300;
+  border-bottom: solid var(--bs-border-color) 1px;
+}
+
+.metricReducer {
+  width: 100%;
+  font-weight: 300;
+}
+
+.metricValues {
+  display: grid;
+  grid-template-columns: max-content max-content;
+  grid-gap: 0 0.3rem;
+}
+
+.metricValue {
+  font-weight: 600;
+}

inspect_ai/_view/www/src/workspace/sidebar/SidebarScoresView.tsx

@@ -0,0 +1,61 @@
+import clsx from "clsx";
+import { Fragment } from "react";
+import { Scores } from "../../types/log";
+import { formatPrettyDecimal } from "../../utils/format";
+import styles from "./SidebarScoresView.module.css";
+
+interface SidebarScoresProps {
+  scores: Scores;
+}
+
+export const SidebarScoresView: React.FC<SidebarScoresProps> = ({ scores }) => {
+  return (
+    <div className={styles.container}>
+      {scores.map((score) => {
+        const name = score.name;
+        const reducer = score.reducer;
+        return (
+          <div className={styles.scoreWrapper}>
+            <div
+              className={clsx(
+                "text-style-secondary",
+                "text-label",
+                "text-size-small",
+                styles.metricName,
+              )}
+            >
+              {name}
+            </div>
+            {reducer ? (
+              <div className={clsx("text-size-small", styles.metricReducer)}>
+                {reducer}
+              </div>
+            ) : (
+              ""
+            )}
+            <div className={clsx("text-size-small", styles.metricValues)}>
+              {Object.keys(score.metrics).map((key) => {
+                const metric = score.metrics[key];
+                return (
+                  <Fragment key={key}>
+                    <div
+                      className={clsx(
+                        "text-style-secondary",
+                        "text-style-label",
+                      )}
+                    >
+                      {metric.name}
+                    </div>
+                    <div className={styles.metricValue}>
+                      {formatPrettyDecimal(metric.value)}
+                    </div>
+                  </Fragment>
+                );
+              })}
+            </div>
+          </div>
+        );
+      })}
+    </div>
+  );
+};

inspect_ai/_view/www/src/workspace/tabs/InfoTab.tsx

@@ -0,0 +1,80 @@
+import { useEffect, useState } from "react";
+import { SampleSummary } from "../../api/types";
+import { MessageBand } from "../../components/MessageBand";
+import { PlanCard } from "../../plan/PlanCard";
+import {
+  EvalError,
+  EvalPlan,
+  EvalResults,
+  EvalSpec,
+  EvalStats,
+} from "../../types/log";
+import { UsageCard } from "../../usage/UsageCard";
+import { TaskErrorCard } from "../error/TaskErrorPanel";
+
+interface PlanTabProps {
+  evalSpec?: EvalSpec;
+  evalPlan?: EvalPlan;
+  evalStats?: EvalStats;
+  evalResults?: EvalResults;
+  samples?: SampleSummary[];
+  evalStatus?: "started" | "error" | "cancelled" | "success";
+  evalError?: EvalError;
+}
+
+export const InfoTab: React.FC<PlanTabProps> = ({
+  evalSpec,
+  evalPlan,
+  evalResults,
+  evalStats,
+  samples,
+  evalStatus,
+  evalError,
+}) => {
+  const [hidden, setHidden] = useState(false);
+  useEffect(() => {
+    setHidden(false);
+  }, [evalSpec, evalPlan, evalResults, evalStats, samples]);
+
+  const infoCards = [];
+  infoCards.push([
+    <PlanCard
+      evalSpec={evalSpec}
+      evalPlan={evalPlan}
+      scores={evalResults?.scores}
+    />,
+  ]);
+
+  if (evalStatus !== "started") {
+    infoCards.push(<UsageCard stats={evalStats} />);
+  }
+
+  // If there is error or progress, includes those within info
+  if (evalStatus === "error" && evalError) {
+    infoCards.unshift(<TaskErrorCard error={evalError} />);
+  }
+
+  const showWarning =
+    (!samples || samples.length === 0) &&
+    evalStatus === "success" &&
+    evalSpec?.dataset.samples &&
+    evalSpec.dataset.samples > 0;
+
+  return (
+    <div style={{ width: "100%" }}>
+      {showWarning ? (
+        <MessageBand
+          message="Unable to display samples (this evaluation log may be too large)."
+          hidden={hidden}
+          setHidden={setHidden}
+          type="warning"
+        />
+      ) : (
+        ""
+      )}
+      <div style={{ padding: "0.5em 1em 0 1em", width: "100%" }}>
+        {infoCards}
+      </div>
+    </div>
+  );
+};

inspect_ai/_view/www/src/workspace/tabs/JsonTab.module.css

@@ -0,0 +1,5 @@
+.jsonTab {
+  padding: 0.5rem;
+  font-size: var(--inspect-font-size-small);
+  width: 100%;
+}

inspect_ai/_view/www/src/workspace/tabs/JsonTab.tsx

@@ -0,0 +1,46 @@
+import { filename } from "../../utils/path";
+
+import { Capabilities } from "../../api/types";
+import { DownloadPanel } from "../../components/DownloadPanel";
+import { JSONPanel } from "../../components/JsonPanel";
+import styles from "./JsonTab.module.css";
+
+const kJsonMaxSize = 10000000;
+
+interface JsonTabProps {
+  logFile?: string;
+  capabilities: Capabilities;
+  selected: boolean;
+  json: string;
+}
+
+/**
+ * Renders JSON tab
+ */
+export const JsonTab: React.FC<JsonTabProps> = ({
+  logFile,
+  capabilities,
+  json,
+}) => {
+  if (logFile && json.length > kJsonMaxSize && capabilities.downloadFiles) {
+    // This JSON file is so large we can't really productively render it
+    // we should instead just provide a DL link
+    const file = `${filename(logFile)}.json`;
+    return (
+      <div className={styles.jsonTab}>
+        <DownloadPanel
+          message="The JSON for this log file is too large to render."
+          buttonLabel="Download JSON File"
+          fileName={file}
+          fileContents={json}
+        />
+      </div>
+    );
+  } else {
+    return (
+      <div className={styles.jsonTab}>
+        <JSONPanel id="task-json-contents" json={json} simple={true} />
+      </div>
+    );
+  }
+};

inspect_ai/_view/www/src/workspace/tabs/SamplesTab.tsx

@@ -0,0 +1,204 @@
+import {
+  Fragment,
+  RefObject,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
+import { SampleSummary } from "../../api/types.ts";
+import { EmptyPanel } from "../../components/EmptyPanel.tsx";
+import { VirtualListRef } from "../../components/VirtualList.tsx";
+import { InlineSampleDisplay } from "../../samples/InlineSampleDisplay";
+import { SampleDialog } from "../../samples/SampleDialog";
+import { SamplesDescriptor } from "../../samples/descriptor/samplesDescriptor.tsx";
+import { SampleList } from "../../samples/list/SampleList";
+import { SampleMode, ScoreFilter } from "../../types.ts";
+import { EvalSample } from "../../types/log";
+import { getSampleProcessor } from "./grouping.ts";
+import { ListItem } from "./types.ts";
+
+interface SamplesTabProps {
+  // Optional props
+  sample?: EvalSample;
+  samples?: SampleSummary[];
+  sampleDescriptor?: SamplesDescriptor;
+  sampleError?: Error;
+
+  // Required props
+  sampleMode: SampleMode;
+  groupBy: "epoch" | "sample" | "none";
+  groupByOrder: "asc" | "desc";
+  sampleStatus: string;
+  selectedSampleIndex: number;
+  setSelectedSampleIndex: (index: number) => void;
+  showingSampleDialog: boolean;
+  setShowingSampleDialog: (showing: boolean) => void;
+  selectedSampleTab?: string;
+  setSelectedSampleTab: (tab: string) => void;
+  epoch: string;
+  filter: ScoreFilter;
+  sampleScrollPositionRef: RefObject<number>;
+  setSampleScrollPosition: (position: number) => void;
+  sampleTabScrollRef: RefObject<HTMLDivElement | null>;
+}
+
+export const SamplesTab: React.FC<SamplesTabProps> = ({
+  sample,
+  samples,
+  sampleMode,
+  groupBy,
+  groupByOrder,
+  sampleDescriptor,
+  sampleStatus,
+  sampleError,
+  selectedSampleIndex,
+  setSelectedSampleIndex,
+  showingSampleDialog,
+  setShowingSampleDialog,
+  selectedSampleTab,
+  setSelectedSampleTab,
+  sampleScrollPositionRef,
+  setSampleScrollPosition,
+  sampleTabScrollRef,
+}) => {
+  const [items, setItems] = useState<ListItem[]>([]);
+  const [sampleItems, setSampleItems] = useState<ListItem[]>([]);
+
+  const sampleListRef = useRef<VirtualListRef>(null);
+  const sampleDialogRef = useRef<HTMLDivElement>(null);
+
+  // Shows the sample dialog
+  const showSample = useCallback(
+    (index: number) => {
+      setSelectedSampleIndex(index);
+      setShowingSampleDialog(true);
+    },
+    [sampleDialogRef],
+  );
+
+  useEffect(() => {
+    if (showingSampleDialog) {
+      setTimeout(() => {
+        sampleDialogRef.current?.focus();
+      }, 0);
+    } else {
+      setTimeout(() => {
+        if (sampleListRef.current) {
+          sampleListRef.current.focus();
+        }
+      }, 0);
+    }
+  }, [showingSampleDialog]);
+
+  useEffect(() => {
+    const sampleProcessor = sampleDescriptor
+      ? getSampleProcessor(
+          samples || [],
+          groupBy,
+          groupByOrder,
+          sampleDescriptor,
+        )
+      : undefined;
+
+    // Process the samples into the proper data structure
+    const items = samples?.flatMap((sample, index) => {
+      const results: ListItem[] = [];
+      const previousSample = index !== 0 ? samples[index - 1] : undefined;
+      const items = sampleProcessor
+        ? sampleProcessor(sample, index, previousSample)
+        : [];
+      results.push(...items);
+      return results;
+    });
+
+    setItems(items || []);
+    setSampleItems(
+      items
+        ? items.filter((item) => {
+            return item.type === "sample";
+          })
+        : [],
+    );
+  }, [samples, groupBy, groupByOrder, sampleDescriptor]);
+
+  const nextSampleIndex = useCallback(() => {
+    if (selectedSampleIndex < sampleItems.length - 1) {
+      return selectedSampleIndex + 1;
+    } else {
+      return -1;
+    }
+  }, [selectedSampleIndex, items]);
+
+  const previousSampleIndex = useCallback(() => {
+    return selectedSampleIndex > 0 ? selectedSampleIndex - 1 : -1;
+  }, [selectedSampleIndex, items]);
+
+  // Manage the next / previous state the selected sample
+  const nextSample = useCallback(() => {
+    const next = nextSampleIndex();
+    if (sampleStatus !== "loading" && next > -1) {
+      setSelectedSampleIndex(next);
+    }
+  }, [selectedSampleIndex, samples, sampleStatus, nextSampleIndex]);
+
+  const previousSample = useCallback(() => {
+    const prev = previousSampleIndex();
+    if (sampleStatus !== "loading" && prev > -1) {
+      setSelectedSampleIndex(prev);
+    }
+  }, [selectedSampleIndex, samples, sampleStatus, previousSampleIndex]);
+
+  const title =
+    selectedSampleIndex > -1 && sampleItems.length > selectedSampleIndex
+      ? sampleItems[selectedSampleIndex].label
+      : "";
+
+  if (!sampleDescriptor) {
+    return <EmptyPanel />;
+  } else {
+    return (
+      <Fragment>
+        {sampleDescriptor && sampleMode === "single" ? (
+          <InlineSampleDisplay
+            id="sample-display"
+            sample={sample}
+            sampleStatus={sampleStatus}
+            sampleError={sampleError}
+            sampleDescriptor={sampleDescriptor}
+            selectedTab={selectedSampleTab}
+            setSelectedTab={setSelectedSampleTab}
+            scrollRef={sampleTabScrollRef}
+          />
+        ) : undefined}
+        {sampleDescriptor && sampleMode === "many" ? (
+          <SampleList
+            listRef={sampleListRef}
+            items={items}
+            sampleDescriptor={sampleDescriptor}
+            selectedIndex={selectedSampleIndex}
+            nextSample={nextSample}
+            prevSample={previousSample}
+            showSample={showSample}
+          />
+        ) : undefined}
+        <SampleDialog
+          id={String(sample?.id || "")}
+          title={title}
+          sample={sample}
+          sampleStatus={sampleStatus}
+          sampleError={sampleError}
+          sampleDescriptor={sampleDescriptor}
+          showingSampleDialog={showingSampleDialog}
+          setShowingSampleDialog={setShowingSampleDialog}
+          selectedTab={selectedSampleTab}
+          setSelectedTab={setSelectedSampleTab}
+          nextSample={nextSample}
+          prevSample={previousSample}
+          sampleScrollPositionRef={sampleScrollPositionRef}
+          setSampleScrollPosition={setSampleScrollPosition}
+        />
+      </Fragment>
+    );
+  }
+};