inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/workspace/WorkSpaceView.tsx

@@ -1,16 +1,15 @@
 import clsx from "clsx";
 import {
+  createElement,
   FC,
   Fragment,
   MouseEvent,
   RefObject,
   useCallback,
-  useMemo,
 } from "react";
-import { SampleSummary } from "../api/types";
+import { RunningMetric } from "../api/types";
 import { EmptyPanel } from "../components/EmptyPanel";
 import { TabPanel, TabSet } from "../components/TabSet";
-import { EvalDescriptor } from "../samples/descriptor/types";
 import {
   EvalPlan,
   EvalResults,
@@ -18,63 +17,37 @@ import {
   EvalStats,
   Status,
 } from "../types/log";
-import { debounce } from "../utils/sync";
 import { Navbar } from "./navbar/Navbar";
 import { TabDescriptor } from "./types";
 
+import { useStore } from "../state/store";
 import styles from "./WorkSpaceView.module.css";
 
 interface WorkSpaceViewProps {
-  logFileName?: string;
   evalSpec: EvalSpec;
   evalPlan?: EvalPlan;
   evalResults?: EvalResults;
+  runningMetrics?: RunningMetric[];
   evalStats?: EvalStats;
-  samples?: SampleSummary[];
-  evalDescriptor?: EvalDescriptor;
   status?: Status;
   showToggle: boolean;
-  tabs: Record<string, TabDescriptor>;
-  selectedTab: string;
-  setSelectedTab: (tab: string) => void;
+  tabs: Record<string, TabDescriptor<any>>;
   divRef: RefObject<HTMLDivElement | null>;
-  offcanvas: boolean;
-  setOffcanvas: (offcanvas: boolean) => void;
-  workspaceTabScrollPositionRef: RefObject<Record<string, number>>;
-  setWorkspaceTabScrollPosition: (tab: string, pos: number) => void;
 }
 
 export const WorkSpaceView: FC<WorkSpaceViewProps> = ({
-  logFileName,
   evalSpec,
   evalPlan,
   evalResults,
+  runningMetrics,
   evalStats,
-  samples,
-  evalDescriptor,
   status,
   showToggle,
-  selectedTab,
   tabs,
-  setSelectedTab,
   divRef,
-  offcanvas,
-  setOffcanvas,
-  workspaceTabScrollPositionRef,
-  setWorkspaceTabScrollPosition,
 }) => {
-  const debouncedScroll = useMemo(() => {
-    return debounce((id, position) => {
-      setWorkspaceTabScrollPosition(id, position);
-    }, 100);
-  }, [setWorkspaceTabScrollPosition]);
-
-  const onScroll = useCallback(
-    (id: string, position: number) => {
-      debouncedScroll(id, position);
-    },
-    [debouncedScroll],
-  );
+  const selectedTab = useStore((state) => state.app.tabs.workspace);
+  const setSelectedTab = useStore((state) => state.appActions.setWorkspaceTab);
 
   const onSelected = useCallback(
     (e: MouseEvent<HTMLElement>) => {
@@ -85,12 +58,6 @@ export const WorkSpaceView: FC<WorkSpaceViewProps> = ({
     },
     [setSelectedTab],
   );
-  const handleScroll = useCallback(
-    (tabid: string, position: number) => {
-      onScroll(tabid, position);
-    },
-    [onScroll],
-  );
 
   if (evalSpec === undefined) {
     return <EmptyPanel />;
@@ -118,14 +85,10 @@ export const WorkSpaceView: FC<WorkSpaceViewProps> = ({
           evalSpec={evalSpec}
           evalPlan={evalPlan}
           evalResults={evalResults}
+          runningMetrics={runningMetrics}
           evalStats={evalStats}
-          samples={samples}
-          evalDescriptor={evalDescriptor}
           status={status}
-          file={logFileName}
           showToggle={showToggle}
-          offcanvas={offcanvas}
-          setOffcanvas={setOffcanvas}
         />
         <div ref={divRef} className={clsx("workspace", styles.workspace)}>
           <div className={clsx("log-detail", styles.tabContainer)}>
@@ -148,14 +111,9 @@ export const WorkSpaceView: FC<WorkSpaceViewProps> = ({
                     selected={selectedTab === tab.id}
                     scrollable={!!tab.scrollable}
                     scrollRef={tab.scrollRef}
-                    scrollPosition={
-                      workspaceTabScrollPositionRef.current?.[tab.id]
-                    }
-                    setScrollPosition={(position: number) => {
-                      handleScroll(tab.id, position);
-                    }}
+                    style={{ height: tab.scrollable ? "100%" : undefined }}
                   >
-                    {tab.content()}
+                    {createElement(tab.component, tab.componentProps)}
                   </TabPanel>
                 );
               })}

inspect_ai/_view/www/src/workspace/navbar/Navbar.tsx

@@ -1,7 +1,7 @@
 import clsx from "clsx";
 import { FC } from "react";
-import { SampleSummary } from "../../api/types";
-import { EvalDescriptor } from "../../samples/descriptor/types";
+import { RunningMetric } from "../../api/types";
+import { useTotalSampleCount } from "../../state/hooks";
 import {
   EvalPlan,
   EvalResults,
@@ -14,16 +14,12 @@ import { PrimaryBar } from "./PrimaryBar";
 import { SecondaryBar } from "./SecondaryBar";
 
 interface NavBarProps {
-  file?: string;
   evalSpec?: EvalSpec;
   evalResults?: EvalResults;
+  runningMetrics?: RunningMetric[];
   evalPlan?: EvalPlan;
   evalStats?: EvalStats;
-  evalDescriptor?: EvalDescriptor;
-  samples?: SampleSummary[];
   status?: Status;
-  offcanvas: boolean;
-  setOffcanvas: (offcanvas: boolean) => void;
   showToggle: boolean;
 }
 
@@ -31,38 +27,32 @@ interface NavBarProps {
  * Renders the Navbar
  */
 export const Navbar: FC<NavBarProps> = ({
-  file,
   evalSpec,
   evalPlan,
   evalResults,
   evalStats,
-  samples,
-  evalDescriptor,
   showToggle,
-  offcanvas,
-  setOffcanvas,
   status,
+  runningMetrics,
 }) => {
+  const totalSampleCount = useTotalSampleCount();
   return (
     <nav className={clsx("navbar", "sticky-top", styles.navbarWrapper)}>
       <PrimaryBar
-        file={file}
         evalSpec={evalSpec}
         evalResults={evalResults}
-        samples={samples}
         showToggle={showToggle}
-        offcanvas={offcanvas}
-        setOffcanvas={setOffcanvas}
         status={status}
+        runningMetrics={runningMetrics}
+        sampleCount={totalSampleCount}
       />
       <SecondaryBar
         evalSpec={evalSpec}
         evalPlan={evalPlan}
         evalResults={evalResults}
         evalStats={evalStats}
-        samples={samples}
-        evalDescriptor={evalDescriptor}
         status={status}
+        sampleCount={totalSampleCount}
       />
     </nav>
   );

inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.module.css

@@ -38,6 +38,7 @@
 }
 
 .taskStatus {
+  display: flex;
   justify-content: end;
   margin-right: 1em;
   margin-bottom: 0;

inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.tsx

@@ -1,41 +1,52 @@
 import clsx from "clsx";
 import { FC, useCallback } from "react";
-import { SampleSummary } from "../../api/types";
+import { RunningMetric } from "../../api/types";
 import { ApplicationIcons } from "../../appearance/icons";
 import { CopyButton } from "../../components/CopyButton";
 import { kModelNone } from "../../constants";
+import { useStore } from "../../state/store";
 import { EvalResults, EvalSpec, Status } from "../../types/log";
 import { filename } from "../../utils/path";
 import styles from "./PrimaryBar.module.css";
-import { ResultsPanel } from "./ResultsPanel";
-import { CancelledPanel, ErroredPanel, RunningPanel } from "./StatusPanel";
+import {
+  displayScorersFromRunningMetrics,
+  ResultsPanel,
+  toDisplayScorers,
+} from "./ResultsPanel";
+import { RunningStatusPanel } from "./RunningStatusPanel";
+import { CancelledPanel, ErroredPanel } from "./StatusPanel";
 
 interface PrimaryBarProps {
   showToggle: boolean;
-  offcanvas: boolean;
-  setOffcanvas: (offcanvas: boolean) => void;
   status?: Status;
   evalResults?: EvalResults;
-  samples?: SampleSummary[];
-  file?: string;
+  runningMetrics?: RunningMetric[];
   evalSpec?: EvalSpec;
+  sampleCount?: number;
 }
 
 export const PrimaryBar: FC<PrimaryBarProps> = ({
   showToggle,
-  offcanvas,
   status,
   evalResults,
-  samples,
-  file,
+  runningMetrics,
   evalSpec,
-  setOffcanvas,
+  sampleCount,
 }) => {
-  const logFileName = file ? filename(file) : "";
+  const offCanvas = useStore((state) => state.app.offcanvas);
+  const setOffCanvas = useStore((state) => state.appActions.setOffcanvas);
+  const streamSamples = useStore((state) => state.capabilities.streamSamples);
+  const selectedLogFile = useStore((state) =>
+    state.logsActions.getSelectedLogFile(),
+  );
+
+  const logFileName = selectedLogFile ? filename(selectedLogFile) : "";
 
   const handleToggle = useCallback(() => {
-    setOffcanvas(!offcanvas);
-  }, [setOffcanvas, offcanvas]);
+    setOffCanvas(!offCanvas);
+  }, [offCanvas, setOffCanvas]);
+
+  const hasRunningMetrics = runningMetrics && runningMetrics.length > 0;
 
   return (
     <div className={clsx(styles.wrapper)}>
@@ -53,7 +64,7 @@ export const PrimaryBar: FC<PrimaryBarProps> = ({
             onClick={handleToggle}
             className={clsx(
               "btn",
-              offcanvas ? "d-md-none" : undefined,
+              offCanvas ? "d-md-none" : undefined,
               styles.toggle,
             )}
             type="button"
@@ -93,22 +104,29 @@ export const PrimaryBar: FC<PrimaryBarProps> = ({
             <div className={clsx("navbar-secondary-text", "text-truncate")}>
               {logFileName}
             </div>
-            {file ? <CopyButton value={file} /> : ""}
+            {selectedLogFile ? <CopyButton value={selectedLogFile} /> : ""}
           </div>
         </div>
       </div>
       <div className={clsx(styles.taskStatus, "navbar-text")}>
-        {status === "success" ? (
-          <ResultsPanel results={evalResults} />
+        {status === "success" ||
+        (status === "started" && streamSamples && hasRunningMetrics) ? (
+          <ResultsPanel
+            scorers={
+              runningMetrics
+                ? displayScorersFromRunningMetrics(runningMetrics)
+                : toDisplayScorers(evalResults?.scores)
+            }
+          />
         ) : undefined}
         {status === "cancelled" ? (
-          <CancelledPanel sampleCount={samples?.length || 0} />
+          <CancelledPanel sampleCount={sampleCount || 0} />
         ) : undefined}
-        {status === "started" ? (
-          <RunningPanel sampleCount={samples?.length || 0} />
+        {status === "started" && (!streamSamples || !hasRunningMetrics) ? (
+          <RunningStatusPanel sampleCount={sampleCount || 0} />
         ) : undefined}
         {status === "error" ? (
-          <ErroredPanel sampleCount={samples?.length || 0} />
+          <ErroredPanel sampleCount={sampleCount || 0} />
         ) : undefined}
       </div>
       <div id="task-created" style={{ display: "none" }}>

inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.module.css

@@ -45,7 +45,6 @@
 
 .multiScorer {
   padding-left: 0;
-  height: 100%;
   display: flex;
   flex-direction: column;
   padding: 0.5em 1em;
@@ -88,3 +87,19 @@
   padding: 0 0.2em;
   justify-content: center;
 }
+
+.moreButton {
+  margin-top: 0.5em;
+  margin-bottom: 0.5em;
+  padding-right: 0;
+}
+
+.metricsSummary {
+  display: flex;
+  flex-direction: column;
+  align-items: flex-end;
+}
+
+.modalScores {
+  padding-bottom: 4em;
+}

inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx

@@ -1,46 +1,115 @@
 import clsx from "clsx";
 import { FC } from "react";
-import { EvalMetric, EvalResults, EvalScore, Reducer } from "../../types/log";
+import { RunningMetric } from "../../api/types";
+import { LinkButton } from "../../components/LinkButton";
+import { Modal } from "../../components/Modal";
+import { useProperty } from "../../state/hooks";
+import { Scores } from "../../types/log";
 import { formatPrettyDecimal } from "../../utils/format";
 import { metricDisplayName } from "../utils";
 import styles from "./ResultsPanel.module.css";
+import { ScoreGrid } from "./ScoreGrid";
 
-interface ResultsPanelProps {
-  results?: EvalResults;
+export interface ResultsMetric {
+  name: string;
+  params?: {};
+  value: number;
 }
 
-interface MetricSummary {
-  reducer: Reducer;
-  metric: EvalMetric;
+export interface ResultsScorer {
+  scorer: string;
+  reducer?: string;
+  metrics: ResultsMetric[];
 }
 
-export const ResultsPanel: FC<ResultsPanelProps> = ({ results }) => {
-  // Map the scores into a list of key/values
-  if (results?.scores?.length === 1) {
-    const scorers: Record<string, MetricSummary[]> = {};
-    results.scores.map((score) => {
-      scorers[score.name] = Object.keys(score.metrics).map((key) => {
+export const displayScorersFromRunningMetrics = (metrics?: RunningMetric[]) => {
+  if (!metrics) {
+    return [];
+  }
+
+  const getKey = (metric: RunningMetric) => {
+    return metric.reducer
+      ? `${metric.scorer}-${metric.reducer}`
+      : metric.scorer;
+  };
+
+  const scorers: Record<string, ResultsScorer> = {};
+  metrics.forEach((metric) => {
+    if (metric.value !== undefined) {
+      const key = getKey(metric);
+      if (scorers[key]) {
+        scorers[key].metrics.push({
+          name: metric.name,
+          value: metric.value,
+        });
+      } else {
+        scorers[key] = {
+          scorer: metric.scorer,
+          reducer: metric.reducer,
+          metrics: [
+            {
+              name: metric.name,
+              value: metric.value,
+            },
+          ],
+        };
+      }
+    }
+  });
+
+  return Object.values(scorers);
+};
+
+export const toDisplayScorers = (scores?: Scores): ResultsScorer[] => {
+  if (!scores) {
+    return [];
+  }
+
+  return scores.map((score) => {
+    return {
+      scorer: score.name,
+      reducer: score.reducer === null ? undefined : score.reducer,
+      metrics: Object.keys(score.metrics).map((key) => {
+        const metric = score.metrics[key];
         return {
-          reducer: score.reducer,
-          metric: {
-            name: key,
-            value: score.metrics[key].value,
-            params: score.metrics[key].params,
-            metadata: {},
-          },
+          name: metric.name,
+          value: metric.value,
+          params: metric.params,
         };
-      });
-    });
+      }),
+    };
+  });
+};
+
+interface ResultsPanelProps {
+  scorers?: ResultsScorer[];
+}
+
+export const ResultsPanel: FC<ResultsPanelProps> = ({ scorers }) => {
+  const [showing, setShowing] = useProperty(
+    "results-panel-metrics",
+    "modal-showing",
+    {
+      defaultValue: false,
+    },
+  );
+
+  if (!scorers || scorers.length === 0) {
+    return undefined;
+  }
 
-    const metrics = Object.values(scorers)[0];
-    const showReducer = metrics && metrics.length > 0 && !!metrics[0].reducer;
+  // Get the display scorers
+  if (scorers.length === 1) {
+    const showReducer = !!scorers[0].reducer;
+    const metrics = scorers[0].metrics;
     return (
       <div className={styles.simpleMetricsRows}>
         {metrics.map((metric, i) => {
           return (
             <VerticalMetric
               key={`simple-metric-${i}`}
-              metricSummary={metric}
+              reducer={scorers[0].reducer}
+              metric={metric}
               isFirst={i === 0}
               showReducer={showReducer}
             />
@@ -49,27 +118,73 @@ export const ResultsPanel: FC<ResultsPanelProps> = ({ results }) => {
       </div>
     );
   } else {
-    const showReducer =
-      results?.scores.findIndex((score) => !!score.reducer) !== -1;
+    const showReducer = scorers.findIndex((score) => !!score.reducer) !== -1;
+    const grouped = groupMetrics(scorers);
+
+    // Try to select metrics with a group size 5 or less, if possible
+    let primaryResults = grouped[0];
+    if (primaryResults.length > 5) {
+      const shorterResults = grouped.find((g) => {
+        return g.length <= 5;
+      });
+      if (shorterResults) {
+        primaryResults = shorterResults;
+      }
+    }
+
     return (
-      <div className={styles.multiMetricsRows}>
-        {results?.scores?.map((score, index) => {
-          return (
-            <MultiScorerMetric
-              key={`multi-metric-${index}`}
-              scorer={score}
-              isFirst={index === 0}
-              showReducer={showReducer}
+      <div className={clsx(styles.metricsSummary)}>
+        <ScoreGrid scoreGroups={[primaryResults]} showReducer={showReducer} />
+        {grouped.length > 1 ? (
+          <>
+            <Modal
+              id="results-metrics"
+              showing={showing}
+              setShowing={setShowing}
+              title={"Scoring Detail"}
+            >
+              <ScoreGrid
+                scoreGroups={grouped}
+                showReducer={showReducer}
+                className={styles.modalScores}
+                striped={false}
+              />
+            </Modal>
+            <LinkButton
+              className={styles.moreButton}
+              text={"All scoring..."}
+              onClick={() => {
+                setShowing(true);
+              }}
             />
-          );
-        })}
+          </>
+        ) : undefined}
       </div>
     );
   }
 };
 
+const metricsKey = (metrics: ResultsMetric[]): string => {
+  const metricKey = metrics.map((m) => m.name).join("");
+  return metricKey;
+};
+
+const groupMetrics = (scorers: ResultsScorer[]): ResultsScorer[][] => {
+  const results: Record<string, ResultsScorer[]> = {};
+  scorers.forEach((scorer) => {
+    if (scorer.metrics.length > 0) {
+      const key = metricsKey(scorer.metrics);
+      results[key] = results[key] || [];
+
+      results[key].push(scorer);
+    }
+  });
+  return Object.values(results);
+};
+
 interface VerticalMetricProps {
-  metricSummary: MetricSummary;
+  metric: ResultsMetric;
+  reducer?: string;
   isFirst: boolean;
   showReducer: boolean;
 }
@@ -77,7 +192,8 @@ interface VerticalMetricProps {
 /** Renders a Vertical Metric
  */
 const VerticalMetric: FC<VerticalMetricProps> = ({
-  metricSummary,
+  metric,
+  reducer,
   isFirst,
   showReducer,
 }) => {
@@ -91,7 +207,7 @@ const VerticalMetric: FC<VerticalMetricProps> = ({
           styles.verticalMetricName,
         )}
       >
-        {metricDisplayName(metricSummary.metric)}
+        {metricDisplayName(metric)}
       </div>
       {showReducer ? (
         <div
@@ -101,7 +217,7 @@ const VerticalMetric: FC<VerticalMetricProps> = ({
             styles.verticalMetricReducer,
           )}
         >
-          {metricSummary.reducer || "default"}
+          {reducer || "default"}
         </div>
       ) : undefined}
 
@@ -112,69 +228,9 @@ const VerticalMetric: FC<VerticalMetricProps> = ({
           styles.verticalMetricValue,
         )}
       >
-        {formatPrettyDecimal(metricSummary.metric.value)}
-      </div>
-    </div>
-  );
-};
-
-interface MultiScorerMetricProps {
-  scorer: EvalScore;
-  isFirst: boolean;
-  showReducer: boolean;
-}
-
-const MultiScorerMetric: FC<MultiScorerMetricProps> = ({
-  scorer,
-  isFirst,
-  showReducer,
-}) => {
-  const titleFontClz = "text-size-base";
-  const reducerFontClz = "text-size-smaller";
-  const valueFontClz = "text-size-base";
-
-  return (
-    <div
-      className={clsx(
-        styles.multiScorer,
-        isFirst ? styles.multiScorerIndent : undefined,
-      )}
-    >
-      <div
-        className={clsx(
-          titleFontClz,
-          "text-style-label",
-          "text-style-secondary",
-          "multi-score-label",
-          styles.multiScorerLabel,
-        )}
-      >
-        {scorer.name}
-      </div>
-      {showReducer ? (
-        <div
-          className={clsx(
-            reducerFontClz,
-            "text-style-label",
-            "text-style-secondary",
-            styles.multiScorerReducer,
-          )}
-        >
-          {scorer.reducer || "default"}
-        </div>
-      ) : undefined}
-      <div className={clsx(valueFontClz, styles.multiScorerValue)}>
-        {Object.keys(scorer.metrics).map((key) => {
-          const metric = scorer.metrics[key];
-          return (
-            <div className={styles.multiScoreMetricGrid} key={key}>
-              <div>{metricDisplayName(metric)}</div>
-              <div className={styles.multiScorerValueContent}>
-                {formatPrettyDecimal(metric.value)}
-              </div>
-            </div>
-          );
-        })}
+        {metric.value !== undefined && metric.value !== null
+          ? formatPrettyDecimal(metric.value)
+          : "n/a"}
       </div>
     </div>
   );