inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/samples/descriptor/score/ScoreDescriptor.tsx

@@ -0,0 +1,99 @@
+import { Value2 } from "../../../types/log";
+import { ScoreDescriptor } from "../types";
+import { booleanScoreDescriptor } from "./BooleanScoreDescriptor";
+import { categoricalScoreDescriptor } from "./CategoricalScoreDescriptor";
+import { numericScoreDescriptor } from "./NumericScoreDescriptor";
+import { objectScoreDescriptor } from "./ObjectScoreDescriptor";
+import { otherScoreDescriptor } from "./OtherScoreDescriptor";
+import { passFailScoreDescriptor } from "./PassFailScoreDescriptor";
+
+type ScorerTypes = string | number | boolean | object;
+interface ScoreCategorizer {
+  describe: (
+    values: Value2[],
+    types?: ScorerTypes[],
+  ) => ScoreDescriptor | undefined;
+}
+
+export const getScoreDescriptorForValues = (
+  uniqScoreValues: Value2[],
+  uniqScoreTypes: ScorerTypes[],
+): ScoreDescriptor | undefined => {
+  for (const categorizer of scoreCategorizers) {
+    const scoreDescriptor = categorizer.describe(
+      uniqScoreValues,
+      uniqScoreTypes,
+    );
+    if (scoreDescriptor) {
+      return scoreDescriptor;
+    }
+  }
+};
+
+const scoreCategorizers: ScoreCategorizer[] = [
+  {
+    describe: (_values: Value2[], types?: ScorerTypes[]) => {
+      if (types && types.length === 1 && types[0] === "boolean") {
+        return booleanScoreDescriptor();
+      }
+    },
+  },
+  {
+    describe: (values: Value2[], _types?: ScorerTypes[]) => {
+      if (
+        values.length === 2 &&
+        values.every((val) => {
+          return val === 1 || val === 0;
+        })
+      ) {
+        return booleanScoreDescriptor();
+      }
+    },
+  },
+  {
+    describe: (values: Value2[], types?: ScorerTypes[]) => {
+      if (
+        types &&
+        types[0] === "string" &&
+        types.length === 1 &&
+        values.length < 5 &&
+        !values.find((val) => {
+          return val !== "I" && val !== "C" && val !== "P" && val !== "N";
+        })
+      ) {
+        return passFailScoreDescriptor(values);
+      }
+    },
+  },
+  {
+    describe: (values: Value2[], types?: ScorerTypes[]) => {
+      if (
+        values.length < 10 &&
+        types &&
+        types.length === 1 &&
+        types[0] === "string"
+      ) {
+        return categoricalScoreDescriptor(values);
+      }
+    },
+  },
+  {
+    describe: (values: Value2[], types?: ScorerTypes[]) => {
+      if (types && types.length !== 0 && types[0] === "number") {
+        return numericScoreDescriptor(values);
+      }
+    },
+  },
+  {
+    describe: (values: Value2[], types?: ScorerTypes[]) => {
+      if (types && types.length !== 0 && types[0] === "object") {
+        return objectScoreDescriptor(values);
+      }
+    },
+  },
+  {
+    describe: (_values: Value2[], _types?: ScorerTypes[]) => {
+      return otherScoreDescriptor();
+    },
+  },
+];

inspect_ai/_view/www/src/samples/descriptor/types.ts

@@ -0,0 +1,55 @@
+import { ReactNode } from "react";
+import { BasicSampleData, SampleSummary } from "../../api/types";
+import { ScoreLabel } from "../../types";
+import { Value2 } from "../../types/log";
+
+export interface EvalDescriptor {
+  epochs: number;
+  samples: SampleSummary[];
+  scores: ScoreLabel[];
+  scoreDescriptor: (scoreLabel: ScoreLabel) => ScoreDescriptor;
+  scorerDescriptor: (
+    sample: BasicSampleData,
+    scoreLabel: ScoreLabel,
+  ) => ScorerDescriptor;
+  score: (
+    sample: BasicSampleData,
+    scoreLabel: ScoreLabel,
+  ) => SelectedScore | undefined;
+  scoreAnswer: (sample: BasicSampleData, scorer: string) => string | undefined;
+}
+
+export interface ScorerDescriptor {
+  metadata: () => Record<string, unknown>;
+  explanation: () => string;
+  answer: () => string;
+  scores: () => Array<{ name: string; rendered: () => ReactNode }>;
+}
+
+export interface ScoreDescriptor {
+  scoreType: string;
+  categories?: Array<Object>;
+  min?: number;
+  max?: number;
+  compare: (a: SelectedScore, b: SelectedScore) => number;
+  render: (score: Value2) => ReactNode;
+}
+
+export interface SelectedScore {
+  value?: Value2;
+  render: () => ReactNode;
+}
+
+export interface MessageShape {
+  raw: MessageShapeData;
+  normalized: MessageShapeData;
+}
+
+export interface MessageShapeData {
+  id: number;
+  input: number;
+  target: number;
+  answer: number;
+  limit: number;
+  score: number;
+}

inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.module.css

@@ -0,0 +1,19 @@
+.flatBody {
+  color: var(--bs-danger);
+  display: grid;
+  grid-template-columns: max-content max-content;
+  column-gap: 0.2em;
+  margin-top: 0.4rem;
+}
+
+.iconSmall {
+  font-size: var(--inspect-font-size-small);
+  line-height: var(--inspect-font-size-small);
+  height: var(--inspect-font-size-small);
+}
+
+.lineBase {
+  font-size: var(--inspect-font-size-base);
+  line-height: var(--inspect-font-size-base);
+  height: var(--inspect-font-size-base);
+}

inspect_ai/_view/www/src/samples/error/FlatSampleErrorView.tsx

@@ -0,0 +1,22 @@
+import { ApplicationIcons } from "../../appearance/icons";
+
+import clsx from "clsx";
+import styles from "./SampleErrorView.module.css";
+import { errorType } from "./error";
+
+interface FlatSampleErrorViewProps {
+  message?: string;
+}
+/**
+ * Component to display a styled error message.
+ */
+export const FlatSampleError: React.FC<FlatSampleErrorViewProps> = ({
+  message,
+}) => {
+  return (
+    <div className={clsx(styles.flatBody)}>
+      <i className={clsx(ApplicationIcons.error, styles.iconSmall)} />
+      <div className={clsx(styles.lineBase)}>{errorType(message)}</div>
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/error/SampleErrorView.module.css

@@ -0,0 +1,17 @@
+.body {
+  color: var(--bs-danger);
+  display: grid;
+  grid-template-columns: 1fr;
+  align-content: align;
+  justify-items: center;
+}
+
+.iconSmall {
+  font-size: var(--inspect-font-size-small);
+  line-height: var(--inspect-font-size-small);
+  height: var(--inspect-font-size-small);
+}
+
+.message {
+  width: 300px;
+}

inspect_ai/_view/www/src/samples/error/SampleErrorView.tsx

@@ -0,0 +1,31 @@
+import { ApplicationIcons } from "../../appearance/icons";
+import { ApplicationStyles } from "../../appearance/styles";
+
+import clsx from "clsx";
+import styles from "./SampleErrorView.module.css";
+import { errorType } from "./error";
+
+interface SampleErrorViewProps {
+  message?: string;
+  align?: string;
+  style?: React.CSSProperties;
+}
+
+/**
+ * Component to display a styled error message.
+ */
+export const SampleErrorView: React.FC<SampleErrorViewProps> = ({
+  message,
+  align,
+}) => {
+  align = align || "center";
+
+  return (
+    <div className={styles.body}>
+      <i className={clsx(ApplicationIcons.error, styles.iconSmall)} />
+      <div className={styles.message} style={ApplicationStyles.lineClamp(2)}>
+        {errorType(message)}
+      </div>
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/error/error.ts

@@ -0,0 +1,15 @@
+/**
+ * Extracts the error type from a given message.
+ * If the message contains parentheses, it returns the substring before the first parenthesis.
+ * Otherwise, it returns "Error".
+ */
+export const errorType = (message?: string): string => {
+  if (!message) {
+    return "Error";
+  }
+
+  if (message.includes("(")) {
+    return message.split("(")[0];
+  }
+  return "Error";
+};

inspect_ai/_view/www/src/samples/list/SampleFooter.module.css

@@ -0,0 +1,9 @@
+.footer {
+  border-top: solid var(--bs-light-border-subtle) 1px;
+  background: var(--bs-light-bg-subtle);
+  display: grid;
+  grid-template-columns: max-content;
+  justify-content: end;
+  align-content: end;
+  padding: 0.2em 1em;
+}

inspect_ai/_view/www/src/samples/list/SampleFooter.tsx

@@ -0,0 +1,14 @@
+interface SampleFooterProps {
+  sampleCount: number;
+}
+
+import clsx from "clsx";
+import styles from "./SampleFooter.module.css";
+
+export const SampleFooter: React.FC<SampleFooterProps> = ({ sampleCount }) => {
+  return (
+    <div className={clsx("text-size-smaller", styles.footer)}>
+      <div>{sampleCount} Samples</div>
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/list/SampleHeader.module.css

@@ -0,0 +1,13 @@
+.header {
+  display: grid;
+  padding-bottom: 0.3em;
+  padding-top: 0.3em;
+  border-bottom: solid var(--bs-light-border-subtle) 1px;
+  grid-gap: 10px;
+  padding-left: 1rem;
+  padding-right: 1rem;
+}
+
+.center {
+  justify-self: center;
+}

inspect_ai/_view/www/src/samples/list/SampleHeader.tsx

@@ -0,0 +1,36 @@
+interface SampleHeaderProps {
+  input?: boolean;
+  target?: boolean;
+  answer?: boolean;
+  limit?: boolean;
+  score?: boolean;
+  gridColumnsTemplate: string;
+}
+import clsx from "clsx";
+import styles from "./SampleHeader.module.css";
+
+export const SampleHeader: React.FC<SampleHeaderProps> = ({
+  input = true,
+  target = true,
+  answer = true,
+  limit = true,
+  score = true,
+  gridColumnsTemplate,
+}) => (
+  <div
+    className={clsx(
+      styles.header,
+      "text-size-smaller",
+      "text-style-label",
+      "text-style-secondary",
+    )}
+    style={{ gridTemplateColumns: gridColumnsTemplate }}
+  >
+    <div>Id</div>
+    <div>{input ? "Input" : ""}</div>
+    <div>{target ? "Target" : ""}</div>
+    <div>{answer ? "Answer" : ""}</div>
+    <div>{limit ? "Limit" : ""}</div>
+    <div className={styles.center}>{score ? "Score" : ""}</div>
+  </div>
+);

inspect_ai/_view/www/src/samples/list/SampleList.module.css

@@ -0,0 +1,11 @@
+.list {
+  flex: 1;
+  overflow-y: auto;
+  outline: none;
+}
+
+.mainLayout {
+  display: flex;
+  flex-direction: column;
+  width: 100%;
+}

inspect_ai/_view/www/src/samples/list/SampleList.tsx

@@ -0,0 +1,247 @@
+import clsx from "clsx";
+import {
+  KeyboardEvent,
+  RefObject,
+  useCallback,
+  useEffect,
+  useMemo,
+  useRef,
+  useState,
+} from "react";
+import { EmptyPanel } from "../../components/EmptyPanel";
+import { MessageBand } from "../../components/MessageBand";
+import { VirtualList, VirtualListRef } from "../../components/VirtualList";
+import { formatNoDecimal } from "../../utils/format";
+import { ListItem } from "../../workspace/tabs/types";
+import { SamplesDescriptor } from "../descriptor/samplesDescriptor";
+import { SampleRow } from "./SampleRow";
+import { SampleSeparator } from "./SampleSeparator";
+
+import { SampleFooter } from "./SampleFooter";
+import { SampleHeader } from "./SampleHeader";
+import styles from "./SampleList.module.css";
+
+const kSampleHeight = 88;
+const kSeparatorHeight = 24;
+
+interface SampleListProps {
+  listRef: RefObject<VirtualListRef | null>;
+  items: ListItem[];
+  sampleDescriptor: SamplesDescriptor;
+  selectedIndex: number;
+  nextSample: () => void;
+  prevSample: () => void;
+  showSample: (index: number) => void;
+  className?: string | string[];
+}
+
+export const SampleList: React.FC<SampleListProps> = (props) => {
+  const {
+    listRef,
+    items,
+    sampleDescriptor,
+    selectedIndex,
+    nextSample,
+    prevSample,
+    showSample,
+    className,
+  } = props;
+
+  // If there are no samples, just display an empty state
+  if (items.length === 0) {
+    return <EmptyPanel>No Samples</EmptyPanel>;
+  }
+
+  const [hidden, setHidden] = useState(false);
+  useEffect(() => {
+    setHidden(false);
+  }, [items]);
+
+  // Keep a mapping of the indexes to items (skipping separators)
+  const itemRowMapping = useMemo(() => {
+    const rowIndexes: number[] = [];
+    items.forEach((item, index) => {
+      if (item.type === "sample") {
+        rowIndexes.push(index);
+      }
+    });
+    return rowIndexes;
+  }, [items]);
+
+  const prevSelectedIndexRef = useRef<number>(null);
+  useEffect(() => {
+    const listEl = listRef.current;
+    if (listEl) {
+      const actualRowIndex = itemRowMapping[selectedIndex];
+
+      const direction =
+        actualRowIndex > (prevSelectedIndexRef.current || 0) ? "down" : "up";
+      listRef.current?.scrollToIndex(actualRowIndex, direction);
+      prevSelectedIndexRef.current = actualRowIndex;
+    }
+  }, [selectedIndex, listRef, itemRowMapping]);
+
+  const renderRow = (item: ListItem) => {
+    if (item.type === "sample") {
+      return (
+        <SampleRow
+          id={`${item.number}`}
+          index={item.index}
+          sample={item.data}
+          height={kSampleHeight}
+          sampleDescriptor={sampleDescriptor}
+          gridColumnsTemplate={gridColumnsValue(sampleDescriptor)}
+          selected={selectedIndex === item.index}
+          showSample={showSample}
+        />
+      );
+    } else if (item.type === "separator") {
+      return (
+        <SampleSeparator
+          id={`sample-group${item.number}`}
+          title={item.data}
+          height={kSeparatorHeight}
+        />
+      );
+    } else {
+      return null;
+    }
+  };
+
+  const onkeydown = useCallback(
+    (e: KeyboardEvent<HTMLDivElement>) => {
+      switch (e.key) {
+        case "ArrowUp":
+          prevSample();
+          e.preventDefault();
+          e.stopPropagation();
+          break;
+        case "ArrowDown":
+          nextSample();
+          e.preventDefault();
+          e.stopPropagation();
+          break;
+        case "Enter":
+          showSample(selectedIndex);
+          e.preventDefault();
+          e.stopPropagation();
+          break;
+      }
+    },
+    [selectedIndex],
+  );
+
+  const { input, limit, answer, target } = gridColumns(sampleDescriptor);
+
+  const sampleCount = items?.reduce((prev, current) => {
+    if (current.type === "sample") {
+      return prev + 1;
+    } else {
+      return prev;
+    }
+  }, 0);
+
+  // Count any sample errors and display a bad alerting the user
+  // to any errors
+  const errorCount = items?.reduce((previous, item: ListItem) => {
+    if (typeof item.data === "object" && item.data.error) {
+      return previous + 1;
+    }
+    return previous;
+  }, 0);
+
+  // Count limits
+  const limitCount = items?.reduce((previous, item) => {
+    if (typeof item.data === "object" && item.data.limit) {
+      return previous + 1;
+    } else {
+      return previous;
+    }
+  }, 0);
+
+  const percentError = (errorCount / sampleCount) * 100;
+  const percentLimit = (limitCount / sampleCount) * 100;
+  const warningMessage =
+    errorCount > 0
+      ? `INFO: ${errorCount} of ${sampleCount} samples (${formatNoDecimal(percentError)}%) had errors and were not scored.`
+      : limitCount
+        ? `INFO: ${limitCount} of ${sampleCount} samples (${formatNoDecimal(percentLimit)}%) completed due to exceeding a limit.`
+        : undefined;
+
+  return (
+    <div className={styles.mainLayout}>
+      {warningMessage ? (
+        <MessageBand
+          message={warningMessage}
+          hidden={hidden}
+          setHidden={setHidden}
+          type="info"
+        />
+      ) : undefined}
+
+      <SampleHeader
+        input={input !== "0"}
+        target={target !== "0"}
+        answer={answer !== "0"}
+        limit={limit !== "0"}
+        gridColumnsTemplate={gridColumnsValue(sampleDescriptor)}
+      />
+      <VirtualList
+        ref={listRef}
+        data={items}
+        tabIndex={0}
+        renderRow={renderRow}
+        onKeyDown={onkeydown}
+        className={clsx(styles.list, className)}
+      />
+      <SampleFooter sampleCount={sampleCount} />
+    </div>
+  );
+};
+
+const gridColumnsValue = (sampleDescriptor: SamplesDescriptor) => {
+  const { input, target, answer, limit, id, score } =
+    gridColumns(sampleDescriptor);
+  return `${id} ${input} ${target} ${answer} ${limit} ${score}`;
+};
+
+const gridColumns = (sampleDescriptor: SamplesDescriptor) => {
+  const input =
+    sampleDescriptor?.messageShape.normalized.input > 0
+      ? Math.max(0.15, sampleDescriptor.messageShape.normalized.input)
+      : 0;
+  const target =
+    sampleDescriptor?.messageShape.normalized.target > 0
+      ? Math.max(0.15, sampleDescriptor.messageShape.normalized.target)
+      : 0;
+  const answer =
+    sampleDescriptor?.messageShape.normalized.answer > 0
+      ? Math.max(0.15, sampleDescriptor.messageShape.normalized.answer)
+      : 0;
+  const limit =
+    sampleDescriptor?.messageShape.normalized.limit > 0
+      ? Math.max(0.15, sampleDescriptor.messageShape.normalized.limit)
+      : 0;
+  const id = Math.max(2, Math.min(10, sampleDescriptor?.messageShape.raw.id));
+  const score = Math.max(
+    3,
+    Math.min(10, sampleDescriptor?.messageShape.raw.score),
+  );
+
+  const frSize = (val: number) => {
+    if (val === 0) {
+      return "0";
+    } else {
+      return `${val}fr`;
+    }
+  };
+
+  return {
+    input: frSize(input),
+    target: frSize(target),
+    answer: frSize(answer),
+    limit: frSize(limit),
+    id: `${id}rem`,
+    score: `${score}rem`,
+  };
+};

inspect_ai/_view/www/src/samples/list/SampleRow.module.css

@@ -0,0 +1,33 @@
+.grid {
+  display: grid;
+  padding-top: 1em;
+  padding-bottom: 1em;
+  border-bottom: solid var(--bs-border-color) 1px;
+  cursor: pointer;
+  overflow-y: hidden;
+  grid-gap: 10px;
+  padding-left: 1rem;
+  padding-right: 1rem;
+}
+
+.selected {
+  box-shadow: inset 0 0 0px 2px var(--bs-focus-ring-color);
+}
+
+.cell {
+  padding-left: 0;
+  padding-right: 0;
+}
+
+.wrapAnywhere {
+  word-wrap: anywhere;
+}
+
+.noLeft {
+  padding-left: 0;
+}
+
+.score {
+  display: flex;
+  justify-self: center;
+}