inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/workspace/tabs/SamplesTab.tsx

@@ -0,0 +1,204 @@
+import {
+  Fragment,
+  RefObject,
+  useCallback,
+  useEffect,
+  useRef,
+  useState,
+} from "react";
+import { SampleSummary } from "../../api/types.ts";
+import { EmptyPanel } from "../../components/EmptyPanel.tsx";
+import { VirtualListRef } from "../../components/VirtualList.tsx";
+import { InlineSampleDisplay } from "../../samples/InlineSampleDisplay";
+import { SampleDialog } from "../../samples/SampleDialog";
+import { SamplesDescriptor } from "../../samples/descriptor/samplesDescriptor.tsx";
+import { SampleList } from "../../samples/list/SampleList";
+import { SampleMode, ScoreFilter } from "../../types.ts";
+import { EvalSample } from "../../types/log";
+import { getSampleProcessor } from "./grouping.ts";
+import { ListItem } from "./types.ts";
+
+interface SamplesTabProps {
+  // Optional props
+  sample?: EvalSample;
+  samples?: SampleSummary[];
+  sampleDescriptor?: SamplesDescriptor;
+  sampleError?: Error;
+
+  // Required props
+  sampleMode: SampleMode;
+  groupBy: "epoch" | "sample" | "none";
+  groupByOrder: "asc" | "desc";
+  sampleStatus: string;
+  selectedSampleIndex: number;
+  setSelectedSampleIndex: (index: number) => void;
+  showingSampleDialog: boolean;
+  setShowingSampleDialog: (showing: boolean) => void;
+  selectedSampleTab?: string;
+  setSelectedSampleTab: (tab: string) => void;
+  epoch: string;
+  filter: ScoreFilter;
+  sampleScrollPositionRef: RefObject<number>;
+  setSampleScrollPosition: (position: number) => void;
+  sampleTabScrollRef: RefObject<HTMLDivElement | null>;
+}
+
+export const SamplesTab: React.FC<SamplesTabProps> = ({
+  sample,
+  samples,
+  sampleMode,
+  groupBy,
+  groupByOrder,
+  sampleDescriptor,
+  sampleStatus,
+  sampleError,
+  selectedSampleIndex,
+  setSelectedSampleIndex,
+  showingSampleDialog,
+  setShowingSampleDialog,
+  selectedSampleTab,
+  setSelectedSampleTab,
+  sampleScrollPositionRef,
+  setSampleScrollPosition,
+  sampleTabScrollRef,
+}) => {
+  const [items, setItems] = useState<ListItem[]>([]);
+  const [sampleItems, setSampleItems] = useState<ListItem[]>([]);
+
+  const sampleListRef = useRef<VirtualListRef>(null);
+  const sampleDialogRef = useRef<HTMLDivElement>(null);
+
+  // Shows the sample dialog
+  const showSample = useCallback(
+    (index: number) => {
+      setSelectedSampleIndex(index);
+      setShowingSampleDialog(true);
+    },
+    [sampleDialogRef],
+  );
+
+  useEffect(() => {
+    if (showingSampleDialog) {
+      setTimeout(() => {
+        sampleDialogRef.current?.focus();
+      }, 0);
+    } else {
+      setTimeout(() => {
+        if (sampleListRef.current) {
+          sampleListRef.current.focus();
+        }
+      }, 0);
+    }
+  }, [showingSampleDialog]);
+
+  useEffect(() => {
+    const sampleProcessor = sampleDescriptor
+      ? getSampleProcessor(
+          samples || [],
+          groupBy,
+          groupByOrder,
+          sampleDescriptor,
+        )
+      : undefined;
+
+    // Process the samples into the proper data structure
+    const items = samples?.flatMap((sample, index) => {
+      const results: ListItem[] = [];
+      const previousSample = index !== 0 ? samples[index - 1] : undefined;
+      const items = sampleProcessor
+        ? sampleProcessor(sample, index, previousSample)
+        : [];
+      results.push(...items);
+      return results;
+    });
+
+    setItems(items || []);
+    setSampleItems(
+      items
+        ? items.filter((item) => {
+            return item.type === "sample";
+          })
+        : [],
+    );
+  }, [samples, groupBy, groupByOrder, sampleDescriptor]);
+
+  const nextSampleIndex = useCallback(() => {
+    if (selectedSampleIndex < sampleItems.length - 1) {
+      return selectedSampleIndex + 1;
+    } else {
+      return -1;
+    }
+  }, [selectedSampleIndex, items]);
+
+  const previousSampleIndex = useCallback(() => {
+    return selectedSampleIndex > 0 ? selectedSampleIndex - 1 : -1;
+  }, [selectedSampleIndex, items]);
+
+  // Manage the next / previous state the selected sample
+  const nextSample = useCallback(() => {
+    const next = nextSampleIndex();
+    if (sampleStatus !== "loading" && next > -1) {
+      setSelectedSampleIndex(next);
+    }
+  }, [selectedSampleIndex, samples, sampleStatus, nextSampleIndex]);
+
+  const previousSample = useCallback(() => {
+    const prev = previousSampleIndex();
+    if (sampleStatus !== "loading" && prev > -1) {
+      setSelectedSampleIndex(prev);
+    }
+  }, [selectedSampleIndex, samples, sampleStatus, previousSampleIndex]);
+
+  const title =
+    selectedSampleIndex > -1 && sampleItems.length > selectedSampleIndex
+      ? sampleItems[selectedSampleIndex].label
+      : "";
+
+  if (!sampleDescriptor) {
+    return <EmptyPanel />;
+  } else {
+    return (
+      <Fragment>
+        {sampleDescriptor && sampleMode === "single" ? (
+          <InlineSampleDisplay
+            id="sample-display"
+            sample={sample}
+            sampleStatus={sampleStatus}
+            sampleError={sampleError}
+            sampleDescriptor={sampleDescriptor}
+            selectedTab={selectedSampleTab}
+            setSelectedTab={setSelectedSampleTab}
+            scrollRef={sampleTabScrollRef}
+          />
+        ) : undefined}
+        {sampleDescriptor && sampleMode === "many" ? (
+          <SampleList
+            listRef={sampleListRef}
+            items={items}
+            sampleDescriptor={sampleDescriptor}
+            selectedIndex={selectedSampleIndex}
+            nextSample={nextSample}
+            prevSample={previousSample}
+            showSample={showSample}
+          />
+        ) : undefined}
+        <SampleDialog
+          id={String(sample?.id || "")}
+          title={title}
+          sample={sample}
+          sampleStatus={sampleStatus}
+          sampleError={sampleError}
+          sampleDescriptor={sampleDescriptor}
+          showingSampleDialog={showingSampleDialog}
+          setShowingSampleDialog={setShowingSampleDialog}
+          selectedTab={selectedSampleTab}
+          setSelectedTab={setSelectedSampleTab}
+          nextSample={nextSample}
+          prevSample={previousSample}
+          sampleScrollPositionRef={sampleScrollPositionRef}
+          setSampleScrollPosition={setSampleScrollPosition}
+        />
+      </Fragment>
+    );
+  }
+};

inspect_ai/_view/www/src/workspace/tabs/grouping.ts

@@ -0,0 +1,195 @@
+import { SampleSummary } from "../../api/types";
+import { SamplesDescriptor } from "../../samples/descriptor/samplesDescriptor";
+import { ListItem, SampleListItem, SeparatorListItem } from "./types";
+
+export const getSampleProcessor = (
+  samples: SampleSummary[],
+  groupBy: "sample" | "epoch" | "none",
+  groupByOrder: "asc" | "desc",
+  sampleDescriptor: SamplesDescriptor,
+): ((
+  sample: SampleSummary,
+  index: number,
+  previousSample?: SampleSummary,
+) => ListItem[]) => {
+  // Perform grouping if there are epochs
+  if (groupBy == "epoch") {
+    return groupByEpoch(samples, sampleDescriptor, groupByOrder);
+  } else if (groupBy === "sample") {
+    return groupBySample(samples, sampleDescriptor, groupByOrder);
+  } else {
+    return noGrouping(samples, groupByOrder);
+  }
+};
+
+/**
+ * Performs no grouping
+ */
+const noGrouping = (
+  samples: SampleSummary[],
+  order: "asc" | "desc",
+): ((sample: SampleSummary, index: number) => ListItem[]) => {
+  const counter = getCounter(samples.length, 1, order);
+  return (sample: SampleSummary, index: number) => {
+    counter.incrementItem();
+    const itemCount = counter.item();
+    return [
+      {
+        label: `Sample ${itemCount}`,
+        number: itemCount,
+        index: index,
+        data: sample,
+        type: "sample",
+      },
+    ];
+  };
+};
+
+/**
+ * Groups by sample (showing separators for Epochs)
+ */
+const groupBySample = (
+  samples: SampleSummary[],
+  sampleDescriptor: SamplesDescriptor,
+  order: "asc" | "desc",
+): ((
+  sample: SampleSummary,
+  index: number,
+  previousSample?: SampleSummary,
+) => ListItem[]) => {
+  // ensure that we are sorted by id
+  samples = samples.sort((a, b) => {
+    if (typeof a.id === "string") {
+      if (order === "asc") {
+        return String(a.id).localeCompare(String(b.id));
+      } else {
+        return String(b.id).localeCompare(String(a.id));
+      }
+    } else {
+      if (order === "asc") {
+        return Number(a.id) - Number(b.id);
+      } else {
+        return Number(b.id) - Number(b.id);
+      }
+    }
+  });
+  const groupCount = samples.length / sampleDescriptor.evalDescriptor.epochs;
+  const itemCount = samples.length / groupCount;
+  const counter = getCounter(itemCount, groupCount, order);
+  return (
+    sample: SampleSummary,
+    index: number,
+    previousSample?: SampleSummary,
+  ): ListItem[] => {
+    const results = [];
+    // Add a separator when the id changes
+    const lastId = previousSample ? previousSample.id : undefined;
+    if (sample.id !== lastId) {
+      counter.incrementGroup();
+      results.push({
+        label: `Sample ${itemCount}`,
+        number: counter.group(),
+        index: index,
+        data: `Sample ${counter.group()}`,
+        type: "separator",
+      } as SeparatorListItem);
+      counter.resetItem();
+    }
+
+    counter.incrementItem();
+    results.push({
+      label: `Sample ${counter.group()} (Epoch ${counter.item()})`,
+      number: counter.item(),
+      index: index,
+      data: sample,
+      type: "sample",
+    } as SampleListItem);
+
+    return results;
+  };
+};
+
+/**
+ * Groups by epoch (showing a separator for each sample)
+ */
+const groupByEpoch = (
+  samples: SampleSummary[],
+  sampleDescriptor: SamplesDescriptor,
+  order: "asc" | "desc",
+): ((
+  sample: SampleSummary,
+  index: number,
+  previousSample?: SampleSummary,
+) => ListItem[]) => {
+  const groupCount = sampleDescriptor.evalDescriptor.epochs;
+  const itemCount = samples.length / groupCount;
+  const counter = getCounter(itemCount, groupCount, order);
+
+  return (
+    sample: SampleSummary,
+    index: number,
+    previousSample?: SampleSummary,
+  ) => {
+    const results = [];
+    const lastEpoch = previousSample ? previousSample.epoch : -1;
+    if (lastEpoch !== sample.epoch) {
+      counter.incrementGroup();
+      // Add a separator
+      results.push({
+        label: `Epoch ${counter.group()}`,
+        number: counter.group(),
+        index: index,
+        data: `Epoch ${counter.group()}`,
+        type: "separator",
+      } as SeparatorListItem);
+      counter.resetItem();
+    }
+
+    // Compute the index within the epoch
+    counter.incrementItem();
+    results.push({
+      label: `Sample ${counter.item()} (Epoch ${counter.group()})`,
+      number: counter.item(),
+      index: index,
+      data: sample,
+      type: "sample",
+    } as SampleListItem);
+
+    return results;
+  };
+};
+
+// An order aware counter that hides increment/decrement behavior
+const getCounter = (
+  itemCount: number,
+  groupCount: number,
+  order: "asc" | "desc",
+) => {
+  let itemIndex = order !== "desc" ? 0 : itemCount + 1;
+  let groupIndex = order !== "desc" ? 0 : groupCount + 1;
+  return {
+    resetItem: () => {
+      itemIndex = order !== "desc" ? 0 : itemCount + 1;
+    },
+    incrementItem: () => {
+      if (order !== "desc") {
+        itemIndex++;
+      } else {
+        itemIndex--;
+      }
+    },
+    incrementGroup: () => {
+      if (order !== "desc") {
+        groupIndex++;
+      } else {
+        groupIndex--;
+      }
+    },
+    item: () => {
+      return itemIndex;
+    },
+    group: () => {
+      return groupIndex;
+    },
+  };
+};

inspect_ai/_view/www/src/workspace/tabs/types.ts

@@ -0,0 +1,19 @@
+import { SampleSummary } from "../../api/types";
+
+export interface SampleListItem {
+  label: string;
+  index: number;
+  number: number;
+  data: SampleSummary;
+  type: "sample";
+}
+
+export interface SeparatorListItem {
+  label: string;
+  index: number;
+  number: number;
+  data: string;
+  type: "separator";
+}
+
+export type ListItem = SampleListItem | SeparatorListItem;

inspect_ai/_view/www/src/workspace/types.ts

@@ -0,0 +1,10 @@
+import { ReactNode, RefObject } from "react";
+
+export interface TabDescriptor {
+  id: string;
+  scrollable: boolean;
+  scrollRef?: RefObject<HTMLDivElement | null>;
+  label: string;
+  content: () => ReactNode;
+  tools?: () => ReactNode[] | undefined;
+}

inspect_ai/_view/www/src/workspace/utils.ts

@@ -0,0 +1,34 @@
+import { EvalMetric } from "../types/log";
+
+export const metricDisplayName = (metric: EvalMetric): string => {
+  let modifier = undefined;
+  for (const metricModifier of metricModifiers) {
+    modifier = metricModifier(metric);
+    if (modifier) {
+      break;
+    }
+  }
+  const metricName = !modifier ? metric.name : `${metric.name}[${modifier}]`;
+
+  return metricName;
+};
+
+type MetricModifier = (metric: EvalMetric) => string | undefined;
+
+const clusterMetricModifier: MetricModifier = (
+  metric: EvalMetric,
+): string | undefined => {
+  if (metric.name !== "stderr") {
+    return undefined;
+  }
+
+  const clusterValue = ((metric.params || {}) as Record<string, unknown>)[
+    "cluster"
+  ];
+  if (clusterValue === undefined || typeof clusterValue !== "string") {
+    return undefined;
+  }
+  return clusterValue;
+};
+
+const metricModifiers: MetricModifier[] = [clusterMetricModifier];

inspect_ai/_view/www/tsconfig.json

@@ -1,10 +1,24 @@
 {
-    "compilerOptions": {
-      "checkJs": true,
-      "target": "es5", // or a different target that suits your needs
-      "downlevelIteration": true,
-      "lib": ["es2019", "dom"], // Add "es2017" or later
-      "esModuleInterop": true
-    },
-    "exclude": ["**/*.mjs"]
-}
+  "compilerOptions": {
+    "target": "ESNext",
+    "lib": ["DOM", "DOM.Iterable", "ESNext"],
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "allowJs": true,
+    "checkJs": true,
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "skipLibCheck": true
+  },
+  "include": ["src"],
+  "exclude": ["node_modules"]
+}

inspect_ai/_view/www/vite.config.js

@@ -1,18 +1,13 @@
+import react from "@vitejs/plugin-react";
 import { defineConfig } from "vite";
-import prism from "vite-plugin-prismjs";
 
 export default defineConfig({
-  base: "", // Set base to an empty string for relative paths
+  mode: "development",
+  base: "",
   build: {
-    // It's important we don't minify, as we check the bundled code into git so
-    // that users don't need to bundle themselves. If we minify, that balloons
-    // the size of the github repo because even minor changes will result in
-    // enormous diffs.
     minify: false,
     rollupOptions: {
       output: {
-        // This configuration ensures a consistent name of output files each
-        // build, which is important given we check them in.
         entryFileNames: `assets/index.js`,
         chunkFileNames: `assets/[name].js`,
         assetFileNames: `assets/[name].[ext]`,
@@ -21,16 +16,12 @@ export default defineConfig({
     sourcemap: true,
   },
   plugins: [
-    prism({
-      languages: ["javascript", "css", "clike", "bash", "python", "python"],
-      css: true,
+    react({
+      jsxRuntime: "automatic",
+      fastRefresh: true,
     }),
   ],
   resolve: {
-    alias: {
-      'react': 'preact/compat',
-      'react-dom': 'preact/compat',
-      'react-dom/test-utils': 'preact/test-utils'
-    }
-  }  
+    dedupe: ["react", "react-dom"],
+  },
 });