inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx

@@ -0,0 +1,180 @@
+import clsx from "clsx";
+import { EvalMetric, EvalResults, EvalScore, Reducer } from "../../types/log";
+import { formatPrettyDecimal } from "../../utils/format";
+import { metricDisplayName } from "../utils";
+import styles from "./ResultsPanel.module.css";
+
+interface ResultsPanelProps {
+  results?: EvalResults;
+}
+
+interface MetricSummary {
+  reducer: Reducer;
+  metric: EvalMetric;
+}
+
+export const ResultsPanel: React.FC<ResultsPanelProps> = ({ results }) => {
+  // Map the scores into a list of key/values
+  if (results?.scores?.length === 1) {
+    const scorers: Record<string, MetricSummary[]> = {};
+    results.scores.map((score) => {
+      scorers[score.name] = Object.keys(score.metrics).map((key) => {
+        return {
+          reducer: score.reducer,
+          metric: {
+            name: key,
+            value: score.metrics[key].value,
+            params: score.metrics[key].params,
+            metadata: {},
+          },
+        };
+      });
+    });
+
+    const metrics = Object.values(scorers)[0];
+    const showReducer = !!metrics[0].reducer;
+    return (
+      <div className={styles.simpleMetricsRows}>
+        {metrics.map((metric, i) => {
+          return (
+            <VerticalMetric
+              key={`simple-metric-${i}`}
+              metricSummary={metric}
+              isFirst={i === 0}
+              showReducer={showReducer}
+            />
+          );
+        })}
+      </div>
+    );
+  } else {
+    const showReducer =
+      results?.scores.findIndex((score) => !!score.reducer) !== -1;
+    return (
+      <div className={styles.multiMetricsRows}>
+        {results?.scores?.map((score, index) => {
+          return (
+            <MultiScorerMetric
+              key={`multi-metric-${index}`}
+              scorer={score}
+              isFirst={index === 0}
+              showReducer={showReducer}
+            />
+          );
+        })}
+      </div>
+    );
+  }
+};
+
+interface VerticalMetricProps {
+  metricSummary: MetricSummary;
+  isFirst: boolean;
+  showReducer: boolean;
+}
+
+/** Renders a Vertical Metric
+ */
+const VerticalMetric: React.FC<VerticalMetricProps> = ({
+  metricSummary,
+  isFirst,
+  showReducer,
+}) => {
+  return (
+    <div style={{ paddingLeft: isFirst ? "0" : "1em" }}>
+      <div
+        className={clsx(
+          "vertical-metric-label",
+          "text-style-label",
+          "text-style-secondary",
+          styles.verticalMetricName,
+        )}
+      >
+        {metricDisplayName(metricSummary.metric)}
+      </div>
+      {showReducer ? (
+        <div
+          className={clsx(
+            "text-style-label",
+            "text-style-secondary",
+            styles.verticalMetricReducer,
+          )}
+        >
+          {metricSummary.reducer || "default"}
+        </div>
+      ) : undefined}
+
+      <div
+        className={clsx(
+          "vertical-metric-value",
+          "text-size-largest",
+          styles.verticalMetricValue,
+        )}
+      >
+        {formatPrettyDecimal(metricSummary.metric.value)}
+      </div>
+    </div>
+  );
+};
+
+interface MultiScorerMetricProps {
+  scorer: EvalScore;
+  isFirst: boolean;
+  showReducer: boolean;
+}
+
+const MultiScorerMetric: React.FC<MultiScorerMetricProps> = ({
+  scorer,
+  isFirst,
+  showReducer,
+}) => {
+  const titleFontClz = "text-size-base";
+  const reducerFontClz = "text-size-smaller";
+  const valueFontClz = "text-size-base";
+
+  return (
+    <div
+      className={clsx(
+        styles.multiScorer,
+        isFirst ? styles.multiScorerIndent : undefined,
+      )}
+    >
+      <div
+        className={clsx(
+          titleFontClz,
+          "text-style-label",
+          "text-style-secondary",
+          "multi-score-label",
+          styles.multiScorerLabel,
+        )}
+      >
+        {scorer.name}
+      </div>
+      {showReducer ? (
+        <div
+          className={clsx(
+            reducerFontClz,
+            "text-style-label",
+            "text-style-secondary",
+            styles.multiScorerReducer,
+          )}
+        >
+          {scorer.reducer || "default"}
+        </div>
+      ) : undefined}
+      <div className={clsx(valueFontClz, styles.multiScorerValue)}>
+        {Object.keys(scorer.metrics).map((key) => {
+          const metric = scorer.metrics[key];
+          return (
+            <div className={styles.multiScoreMetricGrid} key={key}>
+              <div>{metricDisplayName(metric)}</div>
+              <div className={styles.multiScorerValueContent}>
+                {formatPrettyDecimal(metric.value)}
+              </div>
+            </div>
+          );
+        })}
+      </div>
+    </div>
+  );
+};

inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.module.css

@@ -0,0 +1,28 @@
+.staticCol {
+  flex-shrink: 0;
+}
+
+.justifyLeft {
+  justify-self: left;
+}
+
+.justifyCenter {
+  justify-self: center;
+}
+
+.justifyRight {
+  justify-self: right;
+}
+
+.valueGrid {
+  margin: 0;
+  padding: 0.2em 1em 0.2em 1em;
+  display: grid;
+  grid-column-gap: 1em;
+  border-top: 1px solid var(--bs-border-color);
+}
+
+.container {
+  margin: 0;
+  width: 100%;
+}

inspect_ai/_view/www/src/workspace/navbar/SecondaryBar.tsx

@@ -0,0 +1,226 @@
+import clsx from "clsx";
+import { SampleSummary } from "../../api/types";
+import { ExpandablePanel } from "../../components/ExpandablePanel";
+import { LabeledValue } from "../../components/LabeledValue";
+import { EvalDescriptor } from "../../samples/descriptor/types";
+import { scoreFilterItems } from "../../samples/sample-tools/filters";
+import {
+  EvalDataset,
+  EvalPlan,
+  EvalResults,
+  EvalSpec,
+  EvalStats,
+} from "../../types/log";
+import { formatDataset, formatDuration } from "../../utils/format";
+import styles from "./SecondaryBar.module.css";
+
+interface SecondaryBarProps {
+  evalSpec?: EvalSpec;
+  evalPlan?: EvalPlan;
+  evalResults?: EvalResults;
+  evalStats?: EvalStats;
+  evalDescriptor?: EvalDescriptor;
+  samples?: SampleSummary[];
+  status?: string;
+}
+
+/**
+ * Renders the SecondaryBar
+ */
+export const SecondaryBar: React.FC<SecondaryBarProps> = ({
+  evalSpec,
+  evalPlan,
+  evalResults,
+  evalStats,
+  samples,
+  evalDescriptor,
+  status,
+}) => {
+  if (!evalSpec || status !== "success") {
+    return null;
+  }
+
+  const epochs = evalSpec.config.epochs || 1;
+  const hyperparameters: Record<string, unknown> = {
+    ...(evalPlan?.config || {}),
+    ...(evalSpec.task_args || {}),
+  };
+
+  const hasConfig = Object.keys(hyperparameters).length > 0;
+
+  const values = [];
+  values.push({
+    size: "minmax(12%, auto)",
+    value: (
+      <LabeledValue
+        key="sb-dataset"
+        label="Dataset"
+        className={(styles.staticCol, "text-size-small")}
+      >
+        <DatasetSummary
+          dataset={evalSpec.dataset}
+          samples={samples}
+          epochs={epochs}
+        />
+      </LabeledValue>
+    ),
+  });
+
+  const label =
+    evalResults?.scores && evalResults.scores.length > 1 ? "Scorers" : "Scorer";
+  values.push({
+    size: "minmax(12%, auto)",
+    value: (
+      <LabeledValue
+        key="sb-scorer"
+        label={label}
+        className={clsx(
+          styles.staticCol,
+          hasConfig ? styles.justifyLeft : styles.justifyCenter,
+          "text-size-small",
+        )}
+      >
+        <ScorerSummary evalDescriptor={evalDescriptor} />
+      </LabeledValue>
+    ),
+  });
+
+  if (hasConfig) {
+    values.push({
+      size: "minmax(12%, auto)",
+      value: (
+        <LabeledValue
+          key="sb-params"
+          label="Config"
+          className={clsx(styles.justifyRight, "text-size-small")}
+        >
+          <ParamSummary params={hyperparameters} />
+        </LabeledValue>
+      ),
+    });
+  }
+
+  if (evalStats) {
+    const totalDuration = formatDuration(
+      new Date(evalStats?.started_at),
+      new Date(evalStats?.completed_at),
+    );
+    values.push({
+      size: "minmax(12%, auto)",
+      value: (
+        <LabeledValue
+          key="sb-duration"
+          label="Duration"
+          className={clsx(styles.justifyRight, "text-size-small")}
+        >
+          {totalDuration}
+        </LabeledValue>
+      ),
+    });
+  }
+
+  return (
+    <ExpandablePanel
+      className={clsx(styles.container, "text-size-small")}
+      collapse={true}
+      lines={4}
+    >
+      <div
+        className={styles.valueGrid}
+        style={{
+          gridTemplateColumns: `${values
+            .map((val) => {
+              return val.size;
+            })
+            .join(" ")}`,
+        }}
+      >
+        {values.map((val) => {
+          return val.value;
+        })}
+      </div>
+    </ExpandablePanel>
+  );
+};
+
+interface DatasetSummaryProps {
+  dataset?: EvalDataset;
+  samples?: SampleSummary[];
+  epochs: number;
+}
+
+/**
+ * A component that displays the dataset
+ */
+const DatasetSummary: React.FC<DatasetSummaryProps> = ({
+  dataset,
+  samples,
+  epochs,
+}) => {
+  if (!dataset) {
+    return null;
+  }
+
+  return (
+    <div>
+      {samples?.length
+        ? formatDataset(samples.length, epochs, dataset.name)
+        : ""}
+    </div>
+  );
+};
+
+interface ScoreSummaryProps {
+  evalDescriptor?: EvalDescriptor;
+}
+
+/**
+ * A component that displays a list of scrorers
+ */
+const ScorerSummary: React.FC<ScoreSummaryProps> = ({ evalDescriptor }) => {
+  if (!evalDescriptor) {
+    return null;
+  }
+
+  const items = scoreFilterItems(evalDescriptor);
+  return (
+    <span style={{ position: "relative" }}>
+      {Array.from(items).map((item, index, array) => (
+        <span key={index}>
+          <span title={item.tooltip}>{item.canonicalName}</span>
+          {index < array.length - 1 ? ", " : ""}
+        </span>
+      ))}
+    </span>
+  );
+};
+
+interface ParamSummaryProps {
+  params: Record<string, unknown>;
+}
+
+/**
+ * A component that displays a summary of parameters.
+ */
+const ParamSummary: React.FC<ParamSummaryProps> = ({ params }) => {
+  if (!params) {
+    return null;
+  }
+  const paraValues = Object.keys(params).map((key) => {
+    const val = params[key];
+    if (Array.isArray(val) || typeof val === "object") {
+      return `${key}: ${JSON.stringify(val)}`;
+    } else {
+      return `${key}: ${val}`;
+    }
+  });
+  if (paraValues.length > 0) {
+    return (
+      <code style={{ padding: 0, color: "var(--bs-body-color)" }}>
+        {paraValues.join(", ")}
+      </code>
+    );
+  } else {
+    return null;
+  }
+};

inspect_ai/_view/www/src/workspace/navbar/StatusPanel.module.css

@@ -0,0 +1,14 @@
+.statusPanel {
+  padding: 1em;
+  margin-top: 0.5em;
+  text-transform: uppercase;
+  font-size: var(--inspect-font-size-smaller);
+  display: grid;
+  grid-template-columns: auto auto;
+}
+
+.statusIcon {
+  font-size: var(--inspect-font-size-large);
+  margin-right: 0.3em;
+  margin-top: -0.1em;
+}

inspect_ai/_view/www/src/workspace/navbar/StatusPanel.tsx

@@ -0,0 +1,61 @@
+import clsx from "clsx";
+import { ApplicationIcons } from "../../appearance/icons";
+import styles from "./StatusPanel.module.css";
+
+interface StatusProps {
+  sampleCount: number;
+}
+
+export const CancelledPanel: React.FC<StatusProps> = ({ sampleCount }) => {
+  return (
+    <StatusPanel
+      icon={ApplicationIcons.logging["info"]}
+      status="Cancelled"
+      sampleCount={sampleCount}
+    />
+  );
+};
+
+export const ErroredPanel: React.FC<StatusProps> = ({ sampleCount }) => {
+  return (
+    <StatusPanel
+      icon={ApplicationIcons.logging["error"]}
+      status="Task Failed"
+      sampleCount={sampleCount}
+    />
+  );
+};
+
+export const RunningPanel: React.FC<StatusProps> = ({ sampleCount }) => {
+  return (
+    <StatusPanel
+      icon={ApplicationIcons.running}
+      status="Running"
+      sampleCount={sampleCount}
+    />
+  );
+};
+
+interface StatusPanelProps {
+  icon: string;
+  status: string;
+  sampleCount: number;
+}
+
+const StatusPanel: React.FC<StatusPanelProps> = ({
+  icon,
+  status,
+  sampleCount,
+}) => {
+  return (
+    <div className={styles.statusPanel}>
+      <i className={clsx(icon, styles.statusIcon)} style={{}} />
+      <div>
+        <div>${status}</div>
+        <div>
+          (${sampleCount} ${sampleCount === 1 ? "sample" : "samples"})
+        </div>
+      </div>
+    </div>
+  );
+};

inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.module.css

@@ -0,0 +1,15 @@
+.error {
+  color: var(--bs-danger);
+  margin-top: 0.2em;
+}
+
+.running {
+  display: grid;
+  grid-template-columns: max-content max-content;
+  column-gap: 0.5em;
+  margin-top: 0.3em;
+}
+
+.cancelled {
+  margin-top: 0.2em;
+}

inspect_ai/_view/www/src/workspace/sidebar/EvalStatus.tsx

@@ -0,0 +1,71 @@
+import clsx from "clsx";
+import { EvalLogHeader } from "../../api/types";
+import styles from "./EvalStatus.module.css";
+import { SidebarScoreView } from "./SidebarScoreView";
+import { SidebarScoresView } from "./SidebarScoresView";
+
+interface EvalStatusProps {
+  logHeader?: EvalLogHeader;
+}
+
+export const EvalStatus: React.FC<EvalStatusProps> = ({ logHeader }) => {
+  switch (logHeader?.status) {
+    case "error":
+      return <StatusError message="Error" />;
+
+    case "cancelled":
+      return <StatusCancelled message="Cancelled" />;
+
+    case "started":
+      return <StatusRunning message="Running" />;
+
+    default:
+      if (logHeader?.results?.scores && logHeader.results?.scores.length > 0) {
+        if (logHeader.results.scores.length === 1) {
+          return <SidebarScoreView scorer={logHeader.results.scores[0]} />;
+        } else {
+          return <SidebarScoresView scores={logHeader.results.scores} />;
+        }
+      } else {
+        return null;
+      }
+  }
+};
+
+interface StatusProps {
+  message: string;
+}
+
+const StatusCancelled: React.FC<StatusProps> = ({ message }) => {
+  return (
+    <div
+      className={clsx(
+        "text-style-secondary",
+        "text-style-label",
+        "text-size-small",
+        styles.cancelled,
+      )}
+    >
+      {message}
+    </div>
+  );
+};
+
+const StatusRunning: React.FC<StatusProps> = ({ message }) => {
+  return (
+    <div
+      className={clsx(
+        "text-style-secondary",
+        "text-style-label",
+        "text-size-small",
+        styles.running,
+      )}
+    >
+      <div>{message}</div>
+    </div>
+  );
+};
+
+const StatusError: React.FC<StatusProps> = ({ message }) => {
+  return <div className={clsx(styles.error, "text-size-small")}>{message}</div>;
+};

inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.module.css

@@ -0,0 +1,5 @@
+.dirname {
+  overflow: hidden;
+  white-space: nowrap;
+  text-overflow: ellipsis;
+}

inspect_ai/_view/www/src/workspace/sidebar/LogDirectoryTitleView.tsx

@@ -0,0 +1,56 @@
+import clsx from "clsx";
+import styles from "./LogDirectoryTitleView.module.css";
+
+interface LogDirectoryTitleViewProps {
+  log_dir?: string;
+  offcanvas: boolean;
+}
+
+export const LogDirectoryTitleView: React.FC<LogDirectoryTitleViewProps> = ({
+  log_dir,
+  offcanvas,
+}) => {
+  if (log_dir) {
+    const displayDir = prettyDir(log_dir);
+    return (
+      <div style={{ display: "flex", flexDirection: "column" }}>
+        <span
+          className={clsx(
+            "text-style-secondary",
+            "text-style-label",
+            "text-size-small",
+          )}
+        >
+          Log Directory
+        </span>
+        <span
+          title={displayDir}
+          className={clsx("text-size-base", styles.dirname)}
+        >
+          {offcanvas ? displayDir : ""}
+        </span>
+      </div>
+    );
+  } else {
+    return (
+      <span className={clsx("text-size-title")}>
+        {offcanvas ? "Log History" : ""}
+      </span>
+    );
+  }
+};
+
+const prettyDir = (path: string): string => {
+  try {
+    // Try to create a new URL object
+    let url = new URL(path);
+
+    if (url.protocol === "file:") {
+      return url.pathname;
+    } else {
+      return path;
+    }
+  } catch {
+    return path;
+  }
+};