inspect-ai 0.3.81__py3-none-any.whl → 0.3.82__py3-none-any.whl

inspect_ai/_view/www/src/workspace/WorkSpaceView.tsx

@@ -1,16 +1,15 @@
 import clsx from "clsx";
 import {
+  createElement,
   FC,
   Fragment,
   MouseEvent,
   RefObject,
   useCallback,
-  useMemo,
 } from "react";
-import { SampleSummary } from "../api/types";
+import { RunningMetric } from "../api/types";
 import { EmptyPanel } from "../components/EmptyPanel";
 import { TabPanel, TabSet } from "../components/TabSet";
-import { EvalDescriptor } from "../samples/descriptor/types";
 import {
   EvalPlan,
   EvalResults,
@@ -18,63 +17,37 @@ import {
   EvalStats,
   Status,
 } from "../types/log";
-import { debounce } from "../utils/sync";
 import { Navbar } from "./navbar/Navbar";
 import { TabDescriptor } from "./types";
 
+import { useStore } from "../state/store";
 import styles from "./WorkSpaceView.module.css";
 
 interface WorkSpaceViewProps {
-  logFileName?: string;
   evalSpec: EvalSpec;
   evalPlan?: EvalPlan;
   evalResults?: EvalResults;
+  runningMetrics?: RunningMetric[];
   evalStats?: EvalStats;
-  samples?: SampleSummary[];
-  evalDescriptor?: EvalDescriptor;
   status?: Status;
   showToggle: boolean;
-  tabs: Record<string, TabDescriptor>;
-  selectedTab: string;
-  setSelectedTab: (tab: string) => void;
+  tabs: Record<string, TabDescriptor<any>>;
   divRef: RefObject<HTMLDivElement | null>;
-  offcanvas: boolean;
-  setOffcanvas: (offcanvas: boolean) => void;
-  workspaceTabScrollPositionRef: RefObject<Record<string, number>>;
-  setWorkspaceTabScrollPosition: (tab: string, pos: number) => void;
 }
 
 export const WorkSpaceView: FC<WorkSpaceViewProps> = ({
-  logFileName,
   evalSpec,
   evalPlan,
   evalResults,
+  runningMetrics,
   evalStats,
-  samples,
-  evalDescriptor,
   status,
   showToggle,
-  selectedTab,
   tabs,
-  setSelectedTab,
   divRef,
-  offcanvas,
-  setOffcanvas,
-  workspaceTabScrollPositionRef,
-  setWorkspaceTabScrollPosition,
 }) => {
-  const debouncedScroll = useMemo(() => {
-    return debounce((id, position) => {
-      setWorkspaceTabScrollPosition(id, position);
-    }, 100);
-  }, [setWorkspaceTabScrollPosition]);
-
-  const onScroll = useCallback(
-    (id: string, position: number) => {
-      debouncedScroll(id, position);
-    },
-    [debouncedScroll],
-  );
+  const selectedTab = useStore((state) => state.app.tabs.workspace);
+  const setSelectedTab = useStore((state) => state.appActions.setWorkspaceTab);
 
   const onSelected = useCallback(
     (e: MouseEvent<HTMLElement>) => {
@@ -85,12 +58,6 @@ export const WorkSpaceView: FC<WorkSpaceViewProps> = ({
     },
     [setSelectedTab],
   );
-  const handleScroll = useCallback(
-    (tabid: string, position: number) => {
-      onScroll(tabid, position);
-    },
-    [onScroll],
-  );
 
   if (evalSpec === undefined) {
     return <EmptyPanel />;
@@ -118,14 +85,10 @@ export const WorkSpaceView: FC<WorkSpaceViewProps> = ({
           evalSpec={evalSpec}
           evalPlan={evalPlan}
           evalResults={evalResults}
+          runningMetrics={runningMetrics}
           evalStats={evalStats}
-          samples={samples}
-          evalDescriptor={evalDescriptor}
           status={status}
-          file={logFileName}
           showToggle={showToggle}
-          offcanvas={offcanvas}
-          setOffcanvas={setOffcanvas}
         />
         <div ref={divRef} className={clsx("workspace", styles.workspace)}>
           <div className={clsx("log-detail", styles.tabContainer)}>
@@ -148,14 +111,9 @@ export const WorkSpaceView: FC<WorkSpaceViewProps> = ({
                     selected={selectedTab === tab.id}
                     scrollable={!!tab.scrollable}
                     scrollRef={tab.scrollRef}
-                    scrollPosition={
-                      workspaceTabScrollPositionRef.current?.[tab.id]
-                    }
-                    setScrollPosition={(position: number) => {
-                      handleScroll(tab.id, position);
-                    }}
+                    style={{ height: tab.scrollable ? "100%" : undefined }}
                   >
-                    {tab.content()}
+                    {createElement(tab.component, tab.componentProps)}
                   </TabPanel>
                 );
               })}

inspect_ai/_view/www/src/workspace/navbar/Navbar.tsx

@@ -1,7 +1,7 @@
 import clsx from "clsx";
 import { FC } from "react";
-import { SampleSummary } from "../../api/types";
-import { EvalDescriptor } from "../../samples/descriptor/types";
+import { RunningMetric } from "../../api/types";
+import { useTotalSampleCount } from "../../state/hooks";
 import {
   EvalPlan,
   EvalResults,
@@ -14,16 +14,12 @@ import { PrimaryBar } from "./PrimaryBar";
 import { SecondaryBar } from "./SecondaryBar";
 
 interface NavBarProps {
-  file?: string;
   evalSpec?: EvalSpec;
   evalResults?: EvalResults;
+  runningMetrics?: RunningMetric[];
   evalPlan?: EvalPlan;
   evalStats?: EvalStats;
-  evalDescriptor?: EvalDescriptor;
-  samples?: SampleSummary[];
   status?: Status;
-  offcanvas: boolean;
-  setOffcanvas: (offcanvas: boolean) => void;
   showToggle: boolean;
 }
 
@@ -31,38 +27,32 @@ interface NavBarProps {
  * Renders the Navbar
  */
 export const Navbar: FC<NavBarProps> = ({
-  file,
   evalSpec,
   evalPlan,
   evalResults,
   evalStats,
-  samples,
-  evalDescriptor,
   showToggle,
-  offcanvas,
-  setOffcanvas,
   status,
+  runningMetrics,
 }) => {
+  const totalSampleCount = useTotalSampleCount();
   return (
     <nav className={clsx("navbar", "sticky-top", styles.navbarWrapper)}>
       <PrimaryBar
-        file={file}
         evalSpec={evalSpec}
         evalResults={evalResults}
-        samples={samples}
         showToggle={showToggle}
-        offcanvas={offcanvas}
-        setOffcanvas={setOffcanvas}
         status={status}
+        runningMetrics={runningMetrics}
+        sampleCount={totalSampleCount}
       />
       <SecondaryBar
         evalSpec={evalSpec}
         evalPlan={evalPlan}
         evalResults={evalResults}
         evalStats={evalStats}
-        samples={samples}
-        evalDescriptor={evalDescriptor}
         status={status}
+        sampleCount={totalSampleCount}
       />
     </nav>
   );

inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.module.css

@@ -38,6 +38,7 @@
 }
 
 .taskStatus {
+  display: flex;
   justify-content: end;
   margin-right: 1em;
   margin-bottom: 0;

inspect_ai/_view/www/src/workspace/navbar/PrimaryBar.tsx

@@ -1,41 +1,52 @@
 import clsx from "clsx";
 import { FC, useCallback } from "react";
-import { SampleSummary } from "../../api/types";
+import { RunningMetric } from "../../api/types";
 import { ApplicationIcons } from "../../appearance/icons";
 import { CopyButton } from "../../components/CopyButton";
 import { kModelNone } from "../../constants";
+import { useStore } from "../../state/store";
 import { EvalResults, EvalSpec, Status } from "../../types/log";
 import { filename } from "../../utils/path";
 import styles from "./PrimaryBar.module.css";
-import { ResultsPanel } from "./ResultsPanel";
-import { CancelledPanel, ErroredPanel, RunningPanel } from "./StatusPanel";
+import {
+  displayScorersFromRunningMetrics,
+  ResultsPanel,
+  toDisplayScorers,
+} from "./ResultsPanel";
+import { RunningStatusPanel } from "./RunningStatusPanel";
+import { CancelledPanel, ErroredPanel } from "./StatusPanel";
 
 interface PrimaryBarProps {
   showToggle: boolean;
-  offcanvas: boolean;
-  setOffcanvas: (offcanvas: boolean) => void;
   status?: Status;
   evalResults?: EvalResults;
-  samples?: SampleSummary[];
-  file?: string;
+  runningMetrics?: RunningMetric[];
   evalSpec?: EvalSpec;
+  sampleCount?: number;
 }
 
 export const PrimaryBar: FC<PrimaryBarProps> = ({
   showToggle,
-  offcanvas,
   status,
   evalResults,
-  samples,
-  file,
+  runningMetrics,
   evalSpec,
-  setOffcanvas,
+  sampleCount,
 }) => {
-  const logFileName = file ? filename(file) : "";
+  const offCanvas = useStore((state) => state.app.offcanvas);
+  const setOffCanvas = useStore((state) => state.appActions.setOffcanvas);
+  const streamSamples = useStore((state) => state.capabilities.streamSamples);
+  const selectedLogFile = useStore((state) =>
+    state.logsActions.getSelectedLogFile(),
+  );
+
+  const logFileName = selectedLogFile ? filename(selectedLogFile) : "";
 
   const handleToggle = useCallback(() => {
-    setOffcanvas(!offcanvas);
-  }, [setOffcanvas, offcanvas]);
+    setOffCanvas(!offCanvas);
+  }, [offCanvas, setOffCanvas]);
+
+  const hasRunningMetrics = runningMetrics && runningMetrics.length > 0;
 
   return (
     <div className={clsx(styles.wrapper)}>
@@ -53,7 +64,7 @@ export const PrimaryBar: FC<PrimaryBarProps> = ({
             onClick={handleToggle}
             className={clsx(
               "btn",
-              offcanvas ? "d-md-none" : undefined,
+              offCanvas ? "d-md-none" : undefined,
               styles.toggle,
             )}
             type="button"
@@ -93,22 +104,29 @@ export const PrimaryBar: FC<PrimaryBarProps> = ({
             <div className={clsx("navbar-secondary-text", "text-truncate")}>
               {logFileName}
             </div>
-            {file ? <CopyButton value={file} /> : ""}
+            {selectedLogFile ? <CopyButton value={selectedLogFile} /> : ""}
           </div>
         </div>
       </div>
       <div className={clsx(styles.taskStatus, "navbar-text")}>
-        {status === "success" ? (
-          <ResultsPanel results={evalResults} />
+        {status === "success" ||
+        (status === "started" && streamSamples && hasRunningMetrics) ? (
+          <ResultsPanel
+            scorers={
+              runningMetrics
+                ? displayScorersFromRunningMetrics(runningMetrics)
+                : toDisplayScorers(evalResults?.scores)
+            }
+          />
         ) : undefined}
         {status === "cancelled" ? (
-          <CancelledPanel sampleCount={samples?.length || 0} />
+          <CancelledPanel sampleCount={sampleCount || 0} />
         ) : undefined}
-        {status === "started" ? (
-          <RunningPanel sampleCount={samples?.length || 0} />
+        {status === "started" && (!streamSamples || !hasRunningMetrics) ? (
+          <RunningStatusPanel sampleCount={sampleCount || 0} />
         ) : undefined}
         {status === "error" ? (
-          <ErroredPanel sampleCount={samples?.length || 0} />
+          <ErroredPanel sampleCount={sampleCount || 0} />
         ) : undefined}
       </div>
       <div id="task-created" style={{ display: "none" }}>

inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.module.css

@@ -45,7 +45,6 @@
 
 .multiScorer {
   padding-left: 0;
-  height: 100%;
   display: flex;
   flex-direction: column;
   padding: 0.5em 1em;

inspect_ai/_view/www/src/workspace/navbar/ResultsPanel.tsx

@@ -1,46 +1,103 @@
 import clsx from "clsx";
 import { FC } from "react";
-import { EvalMetric, EvalResults, EvalScore, Reducer } from "../../types/log";
+import { RunningMetric } from "../../api/types";
+import { Scores } from "../../types/log";
 import { formatPrettyDecimal } from "../../utils/format";
 import { metricDisplayName } from "../utils";
 import styles from "./ResultsPanel.module.css";
 
-interface ResultsPanelProps {
-  results?: EvalResults;
+export interface ResultsMetric {
+  name: string;
+  params?: {};
+  value: number;
 }
 
-interface MetricSummary {
-  reducer: Reducer;
-  metric: EvalMetric;
+export interface ResultsScorer {
+  scorer: string;
+  reducer?: string;
+  metrics: ResultsMetric[];
 }
 
-export const ResultsPanel: FC<ResultsPanelProps> = ({ results }) => {
-  // Map the scores into a list of key/values
-  if (results?.scores?.length === 1) {
-    const scorers: Record<string, MetricSummary[]> = {};
-    results.scores.map((score) => {
-      scorers[score.name] = Object.keys(score.metrics).map((key) => {
+export const displayScorersFromRunningMetrics = (metrics?: RunningMetric[]) => {
+  if (!metrics) {
+    return [];
+  }
+
+  const getKey = (metric: RunningMetric) => {
+    return metric.reducer
+      ? `${metric.scorer}-${metric.reducer}`
+      : metric.scorer;
+  };
+
+  const scorers: Record<string, ResultsScorer> = {};
+  metrics.forEach((metric) => {
+    if (metric.value !== undefined) {
+      const key = getKey(metric);
+      if (scorers[key]) {
+        scorers[key].metrics.push({
+          name: metric.name,
+          value: metric.value,
+        });
+      } else {
+        scorers[key] = {
+          scorer: metric.scorer,
+          reducer: metric.reducer,
+          metrics: [
+            {
+              name: metric.name,
+              value: metric.value,
+            },
+          ],
+        };
+      }
+    }
+  });
+
+  return Object.values(scorers);
+};
+
+export const toDisplayScorers = (scores?: Scores): ResultsScorer[] => {
+  if (!scores) {
+    return [];
+  }
+
+  return scores.map((score) => {
+    return {
+      scorer: score.name,
+      reducer: score.reducer === null ? undefined : score.reducer,
+      metrics: Object.keys(score.metrics).map((key) => {
+        const metric = score.metrics[key];
         return {
-          reducer: score.reducer,
-          metric: {
-            name: key,
-            value: score.metrics[key].value,
-            params: score.metrics[key].params,
-            metadata: {},
-          },
+          name: metric.name,
+          value: metric.value,
+          params: metric.params,
         };
-      });
-    });
+      }),
+    };
+  });
+};
+
+interface ResultsPanelProps {
+  scorers?: ResultsScorer[];
+}
+
+export const ResultsPanel: FC<ResultsPanelProps> = ({ scorers }) => {
+  if (!scorers || scorers.length === 0) {
+    return undefined;
+  }
 
-    const metrics = Object.values(scorers)[0];
-    const showReducer = metrics && metrics.length > 0 && !!metrics[0].reducer;
+  // Get the display scorers
+  if (scorers.length === 1) {
+    const showReducer = !!scorers[0].reducer;
+    const metrics = scorers[0].metrics;
     return (
       <div className={styles.simpleMetricsRows}>
         {metrics.map((metric, i) => {
           return (
             <VerticalMetric
               key={`simple-metric-${i}`}
-              metricSummary={metric}
+              reducer={scorers[0].reducer}
+              metric={metric}
               isFirst={i === 0}
               showReducer={showReducer}
             />
@@ -49,15 +106,14 @@ export const ResultsPanel: FC<ResultsPanelProps> = ({ results }) => {
       </div>
     );
   } else {
-    const showReducer =
-      results?.scores.findIndex((score) => !!score.reducer) !== -1;
+    const showReducer = scorers.findIndex((score) => !!score.reducer) !== -1;
     return (
       <div className={styles.multiMetricsRows}>
-        {results?.scores?.map((score, index) => {
+        {scorers.map((scorer, index) => {
           return (
             <MultiScorerMetric
               key={`multi-metric-${index}`}
-              scorer={score}
+              scorer={scorer}
               isFirst={index === 0}
               showReducer={showReducer}
             />
@@ -69,7 +125,8 @@ export const ResultsPanel: FC<ResultsPanelProps> = ({ results }) => {
 };
 
 interface VerticalMetricProps {
-  metricSummary: MetricSummary;
+  metric: ResultsMetric;
+  reducer?: string;
   isFirst: boolean;
   showReducer: boolean;
 }
@@ -77,7 +134,8 @@ interface VerticalMetricProps {
 /** Renders a Vertical Metric
  */
 const VerticalMetric: FC<VerticalMetricProps> = ({
-  metricSummary,
+  metric,
+  reducer,
   isFirst,
   showReducer,
 }) => {
@@ -91,7 +149,7 @@ const VerticalMetric: FC<VerticalMetricProps> = ({
           styles.verticalMetricName,
         )}
       >
-        {metricDisplayName(metricSummary.metric)}
+        {metricDisplayName(metric)}
       </div>
       {showReducer ? (
         <div
@@ -101,7 +159,7 @@ const VerticalMetric: FC<VerticalMetricProps> = ({
             styles.verticalMetricReducer,
           )}
         >
-          {metricSummary.reducer || "default"}
+          {reducer || "default"}
         </div>
       ) : undefined}
 
@@ -112,14 +170,16 @@ const VerticalMetric: FC<VerticalMetricProps> = ({
           styles.verticalMetricValue,
         )}
       >
-        {formatPrettyDecimal(metricSummary.metric.value)}
+        {metric.value !== undefined && metric.value !== null
+          ? formatPrettyDecimal(metric.value)
+          : "n/a"}
       </div>
     </div>
   );
 };
 
 interface MultiScorerMetricProps {
-  scorer: EvalScore;
+  scorer: ResultsScorer;
   isFirst: boolean;
   showReducer: boolean;
 }
@@ -149,7 +209,7 @@ const MultiScorerMetric: FC<MultiScorerMetricProps> = ({
           styles.multiScorerLabel,
         )}
       >
-        {scorer.name}
+        {scorer.scorer}
       </div>
       {showReducer ? (
         <div
@@ -164,13 +224,12 @@ const MultiScorerMetric: FC<MultiScorerMetricProps> = ({
         </div>
       ) : undefined}
       <div className={clsx(valueFontClz, styles.multiScorerValue)}>
-        {Object.keys(scorer.metrics).map((key) => {
-          const metric = scorer.metrics[key];
+        {scorer.metrics.map((metric) => {
           return (
-            <div className={styles.multiScoreMetricGrid} key={key}>
+            <div className={styles.multiScoreMetricGrid} key={metric.name}>
               <div>{metricDisplayName(metric)}</div>
               <div className={styles.multiScorerValueContent}>
-                {formatPrettyDecimal(metric.value)}
+                {metric.value ? formatPrettyDecimal(metric.value) : undefined}
               </div>
             </div>
           );

inspect_ai/_view/www/src/workspace/navbar/RunningStatusPanel.module.css

@@ -0,0 +1,32 @@
+.statusContainer {
+  display: inline-block;
+  margin-top: 0.3em;
+}
+
+.status {
+  display: grid;
+  grid-template-columns: auto auto;
+}
+
+.statusText {
+  margin-top: 0.2em;
+}
+
+.metricsRows {
+  margin-top: 0.3em;
+  margin-left: 1.25em;
+  display: grid;
+  grid-template-columns: auto auto;
+  grid-template-rows: auto;
+  column-gap: 0.5em;
+}
+
+.icon {
+  font-size: var(--inspect-font-size-large);
+  margin-right: 0.3em;
+  margin-top: -0.1em;
+}
+
+.value {
+  font-weight: 600;
+}

inspect_ai/_view/www/src/workspace/navbar/RunningStatusPanel.tsx

@@ -0,0 +1,32 @@
+import clsx from "clsx";
+import { RunningMetric } from "../../api/types";
+import { ApplicationIcons } from "../../appearance/icons";
+
+import { FC } from "react";
+import styles from "./RunningStatusPanel.module.css";
+
+export interface RunningPanelProps {
+  sampleCount: number;
+  displayMetrics?: RunningMetric[];
+}
+
+export const RunningStatusPanel: FC<RunningPanelProps> = ({ sampleCount }) => {
+  return (
+    <div>
+      <div className={clsx(styles.statusContainer)}>
+        <div className={clsx(styles.status)}>
+          <i className={clsx(ApplicationIcons.running, styles.icon)} />
+          <div
+            className={clsx(
+              styles.statusText,
+              "text-style-label",
+              "text-size-smaller",
+            )}
+          >
+            Running ({sampleCount} samples)
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};