inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/samples/descriptor/samplesDescriptor.tsx

@@ -0,0 +1,365 @@
+import { ReactNode } from "react";
+import { BasicSampleData, SampleSummary } from "../../api/types";
+import { ScoreLabel } from "../../types";
+import { Value2 } from "../../types/log";
+import { arrayToString, inputString } from "../../utils/format";
+import { getScoreDescriptorForValues } from "./score/ScoreDescriptor";
+import {
+  EvalDescriptor,
+  MessageShape,
+  ScoreDescriptor,
+  ScorerDescriptor,
+  SelectedScore,
+} from "./types";
+
+export interface SamplesDescriptor {
+  evalDescriptor: EvalDescriptor;
+  messageShape: MessageShape;
+  selectedScoreDescriptor?: ScoreDescriptor;
+  selectedScore: (sample: BasicSampleData) => SelectedScore | undefined;
+  selectedScorerDescriptor: (sample: BasicSampleData) => ScorerDescriptor;
+}
+
+export const createEvalDescriptor = (
+  scores: ScoreLabel[],
+  epochs: number,
+  samples?: SampleSummary[],
+): EvalDescriptor | undefined => {
+  if (!samples) {
+    return undefined;
+  }
+
+  const scoreValue = (
+    sample: BasicSampleData,
+    scoreLabel?: ScoreLabel,
+  ): Value2 | undefined => {
+    // no scores, no value
+    if (
+      sample.scores === null ||
+      Object.keys(sample.scores).length === 0 ||
+      !scoreLabel
+    ) {
+      return undefined;
+    }
+
+    if (
+      scoreLabel.scorer !== scoreLabel.name &&
+      sample.scores[scoreLabel.scorer] &&
+      sample.scores[scoreLabel.scorer].value
+    ) {
+      return sample.scores[scoreLabel.scorer].value;
+    } else if (sample.scores[scoreLabel.name]) {
+      return sample.scores[scoreLabel.name].value;
+    } else {
+      return undefined;
+    }
+  };
+
+  const scoreAnswer = (
+    sample: BasicSampleData,
+    scorer: string,
+  ): string | undefined => {
+    if (sample && sample.scores) {
+      const sampleScore = sample.scores[scorer];
+      if (sampleScore && sampleScore.answer) {
+        return sampleScore.answer;
+      }
+    } else {
+      return undefined;
+    }
+  };
+
+  const scoreExplanation = (
+    sample: BasicSampleData,
+    scorer: string,
+  ): string | undefined => {
+    if (sample && sample.scores) {
+      const sampleScore = sample.scores[scorer];
+      if (sampleScore && sampleScore.explanation) {
+        return sampleScore.explanation;
+      }
+    }
+    return undefined;
+  };
+
+  // Retrieve the metadata for a sample
+  const scoreMetadata = (
+    sample: BasicSampleData,
+    scorer: string,
+  ): Record<string, unknown> | undefined => {
+    if (sample && sample.scores) {
+      const sampleScore = sample.scores[scorer];
+      if (sampleScore && sampleScore.metadata) {
+        return sampleScore.metadata;
+      }
+    }
+    return undefined;
+  };
+
+  const scoreDescriptorMap: Record<string, ScoreDescriptor> = {};
+  for (const scoreLabel of scores) {
+    const uniqScoreValues = [
+      ...new Set(
+        samples
+          .filter((sample) => !!sample.scores)
+          .filter((sample) => {
+            // There is no selected scorer, so include this value
+            if (!scoreLabel) {
+              return true;
+            }
+
+            // There are no scores, so exclude this
+            if (!sample.scores) {
+              return false;
+            }
+
+            if (scoreLabel.scorer !== scoreLabel.name) {
+              return (
+                Object.keys(sample.scores).includes(scoreLabel.scorer) &&
+                Object.keys(sample.scores[scoreLabel.scorer].value).includes(
+                  scoreLabel.name,
+                )
+              );
+            } else {
+              return Object.keys(sample.scores).includes(scoreLabel.name);
+            }
+          })
+          .map((sample) => {
+            return scoreValue(sample, scoreLabel);
+          })
+          .filter((value) => {
+            return value !== null;
+          })
+          .filter((value) => {
+            return value !== undefined;
+          }),
+      ),
+    ];
+    const uniqScoreTypes = [
+      ...new Set(uniqScoreValues.map((scoreValue) => typeof scoreValue)),
+    ];
+
+    const scoreDescriptor = getScoreDescriptorForValues(
+      uniqScoreValues,
+      uniqScoreTypes,
+    );
+    if (scoreDescriptor) {
+      scoreDescriptorMap[scoreLabelKey(scoreLabel)] = scoreDescriptor;
+    }
+  }
+
+  const scoreDescriptor = (scoreLabel: ScoreLabel): ScoreDescriptor => {
+    return scoreDescriptorMap[scoreLabelKey(scoreLabel)];
+  };
+
+  const scoreRendered = (
+    sample: BasicSampleData,
+    scoreLabel: ScoreLabel,
+  ): ReactNode => {
+    const descriptor = scoreDescriptor(scoreLabel);
+    const score = scoreValue(sample, scoreLabel);
+    if (score === null) {
+      return "null";
+    } else if (score === undefined) {
+      return "";
+    } else if (score && descriptor && descriptor.render) {
+      return descriptor.render(score);
+    } else {
+      return <span>{String(score)}</span>;
+    }
+  };
+
+  const scorerDescriptor = (
+    sample: BasicSampleData,
+    scoreLabel: ScoreLabel,
+  ): ScorerDescriptor => {
+    return {
+      metadata: () => {
+        return scoreMetadata(sample, scoreLabel.scorer) || {};
+      },
+      explanation: () => {
+        return scoreExplanation(sample, scoreLabel.scorer) || "";
+      },
+      answer: () => {
+        return scoreAnswer(sample, scoreLabel.scorer) || "";
+      },
+      scores: () => {
+        if (!sample || !sample.scores) {
+          return [];
+        }
+        const myScoreDescriptor = scoreDescriptor(scoreLabel);
+        if (!myScoreDescriptor) {
+          return [];
+        }
+
+        // Make a list of all the valid score names (this is
+        // used to distinguish between dictionaries that contain
+        // scores that should be treated as standlone scores and
+        // dictionaries that just contain random values, which is allowed)
+        const scoreNames = scores.map((score) => {
+          return score.name;
+        });
+        const sampleScorer = sample.scores[scoreLabel.scorer];
+        const scoreVal = sampleScorer.value as Value2;
+
+        if (typeof scoreVal === "object") {
+          const names = Object.keys(scoreVal);
+
+          // See if this is a dictionary of score names
+          // if any of the score names match, treat it
+          // as a scorer dictionary
+          if (
+            names.find((name) => {
+              return scoreNames.includes(name);
+            })
+          ) {
+            // Since this dictionary contains keys which are  scores
+            // we actually render the individual scores
+            const scores = names.map((name) => {
+              return {
+                name,
+                rendered: () => {
+                  return myScoreDescriptor.render(scoreVal);
+                },
+              };
+            });
+            return scores;
+          } else {
+            // Since this dictionary contains keys which are not scores
+            // we just treat it like an opaque dictionary
+            return [
+              {
+                name: scoreLabel.scorer,
+                rendered: () => {
+                  return myScoreDescriptor.render(scoreVal);
+                },
+              },
+            ];
+          }
+        } else {
+          return [
+            {
+              name: scoreLabel.scorer,
+              rendered: () => {
+                return myScoreDescriptor.render(scoreVal);
+              },
+            },
+          ];
+        }
+      },
+    };
+  };
+
+  const score = (
+    sample: BasicSampleData,
+    scoreLabel: ScoreLabel,
+  ): SelectedScore => {
+    return {
+      value: scoreValue(sample, scoreLabel),
+      render: () => {
+        return scoreRendered(sample, scoreLabel);
+      },
+    };
+  };
+
+  return {
+    epochs,
+    samples,
+    scores,
+    scorerDescriptor,
+    scoreDescriptor,
+    score,
+    scoreAnswer,
+  };
+};
+
+export const createSamplesDescriptor = (
+  evalDescriptor: EvalDescriptor,
+  selectedScore: ScoreLabel,
+): SamplesDescriptor | undefined => {
+  // Find the total length of the value so we can compute an average
+  const sizes = evalDescriptor.samples.reduce(
+    (previous, current) => {
+      const text = inputString(current.input).join(" ");
+      const score = evalDescriptor.score(current, selectedScore);
+      const scoreValue = score?.value;
+      const scoreText = scoreValue
+        ? String(scoreValue)
+        : current.error
+          ? String(current.error)
+          : "";
+      previous[0] = Math.min(Math.max(previous[0], text.length), 300);
+      previous[1] = Math.min(
+        Math.max(previous[1], arrayToString(current.target).length),
+        300,
+      );
+      previous[2] = Math.min(
+        Math.max(
+          previous[2],
+          evalDescriptor.scoreAnswer(current, selectedScore?.name)?.length || 0,
+        ),
+        300,
+      );
+      previous[3] = Math.min(
+        Math.max(previous[3], current.limit ? current.limit.length : 0),
+        50,
+      );
+      previous[4] = Math.min(
+        Math.max(previous[4], String(current.id).length),
+        10,
+      );
+      previous[5] = Math.min(Math.max(previous[5], scoreText.length), 30);
+
+      return previous;
+    },
+    [0, 0, 0, 0, 0, 0],
+  );
+
+  // normalize to base 1
+  const maxSizes = {
+    input: Math.min(sizes[0], 300),
+    target: Math.min(sizes[1], 300),
+    answer: Math.min(sizes[2], 300),
+    limit: Math.min(sizes[3], 50),
+    id: Math.min(sizes[4], 10),
+    score: Math.min(sizes[4], 30),
+  };
+  const base =
+    maxSizes.input +
+      maxSizes.target +
+      maxSizes.answer +
+      maxSizes.limit +
+      maxSizes.id +
+      maxSizes.score || 1;
+  const messageShape = {
+    raw: {
+      input: sizes[0],
+      target: sizes[1],
+      answer: sizes[2],
+      limit: sizes[3],
+      id: sizes[4],
+      score: sizes[5],
+    },
+    normalized: {
+      input: maxSizes.input / base,
+      target: maxSizes.target / base,
+      answer: maxSizes.answer / base,
+      limit: maxSizes.limit / base,
+      id: maxSizes.id / base,
+      score: maxSizes.score / base,
+    },
+  };
+
+  return {
+    evalDescriptor,
+    messageShape,
+    selectedScoreDescriptor: evalDescriptor.scoreDescriptor(selectedScore),
+    selectedScore: (sample) => evalDescriptor.score(sample, selectedScore),
+    selectedScorerDescriptor: (sample) =>
+      evalDescriptor.scorerDescriptor(sample, selectedScore),
+  };
+};
+
+const scoreLabelKey = (scoreLabel: ScoreLabel) => {
+  return `${scoreLabel?.scorer}.${scoreLabel.name}`;
+};

inspect_ai/_view/www/src/samples/descriptor/score/BooleanScoreDescriptor.module.css

@@ -0,0 +1,22 @@
+.circle {
+  font-family: "Consola Regular";
+  width: 20px;
+  height: 20px;
+  display: inline-flex;
+  justify-content: center;
+  align-items: center;
+  border-radius: 50%;
+  padding-top: 1px;
+}
+
+.green {
+  background-color: var(--bs-success);
+  border-color: var(--bs-success);
+  color: var(--bs-body-bg);
+}
+
+.red {
+  background-color: var(--bs-danger);
+  border-color: var(--bs-danger);
+  color: var(--bs-body-bg);
+}

inspect_ai/_view/www/src/samples/descriptor/score/BooleanScoreDescriptor.tsx

@@ -0,0 +1,26 @@
+import clsx from "clsx";
+import { Value2 } from "../../../types/log";
+import { ScoreDescriptor, SelectedScore } from "../types";
+import styles from "./BooleanScoreDescriptor.module.css";
+
+export const booleanScoreDescriptor = (): ScoreDescriptor => {
+  return {
+    scoreType: "boolean",
+    compare: (a: SelectedScore, b: SelectedScore) => {
+      return Number(a.value) - Number(b.value);
+    },
+    render: (score: Value2) => {
+      return (
+        <span
+          className={clsx(
+            styles.circle,
+            "text-size-small",
+            score ? "green" : "red",
+          )}
+        >
+          {String(score)}
+        </span>
+      );
+    },
+  };
+};

inspect_ai/_view/www/src/samples/descriptor/score/CategoricalScoreDescriptor.tsx

@@ -0,0 +1,18 @@
+import { kScoreTypeCategorical } from "../../../constants";
+import { Value2 } from "../../../types/log";
+import { ScoreDescriptor } from "../types";
+
+export const categoricalScoreDescriptor = (
+  values: Value2[],
+): ScoreDescriptor => {
+  return {
+    scoreType: kScoreTypeCategorical,
+    categories: values,
+    compare: (a, b) => {
+      return String(a).localeCompare(String(b));
+    },
+    render: (score) => {
+      return String(score);
+    },
+  };
+};

inspect_ai/_view/www/src/samples/descriptor/score/NumericScoreDescriptor.tsx

@@ -0,0 +1,27 @@
+import { kScoreTypeNumeric } from "../../../constants";
+import { Value2 } from "../../../types/log";
+import { formatDecimalNoTrailingZeroes } from "../../../utils/format";
+import { ScoreDescriptor } from "../types";
+
+export const numericScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
+  const onlyNumeric = values.filter((val) => {
+    return typeof val === "number";
+  });
+
+  return {
+    scoreType: kScoreTypeNumeric,
+    min: Math.min(...onlyNumeric),
+    max: Math.max(...onlyNumeric),
+    compare: (a, b) => {
+      if (typeof a === "number" && typeof b === "number") {
+        return a - b;
+      } else {
+        console.warn("Comparing non-numerics using a numeric score descriptor");
+        return 0;
+      }
+    },
+    render: (score) => {
+      return formatDecimalNoTrailingZeroes(Number(score));
+    },
+  };
+};

inspect_ai/_view/www/src/samples/descriptor/score/ObjectScoreDescriptor.module.css

@@ -0,0 +1,18 @@
+.container {
+  display: flex;
+  flex-direction: column;
+  align-items: center;
+  margin-left: 0.5rem;
+}
+
+.padded {
+  padding-bottom: 1em;
+}
+
+.key {
+  font-weight: 300;
+}
+
+.value {
+  font-weight: 600;
+}

inspect_ai/_view/www/src/samples/descriptor/score/ObjectScoreDescriptor.tsx

@@ -0,0 +1,71 @@
+import clsx from "clsx";
+import { JSX } from "react";
+import { kScoreTypeObject } from "../../../constants";
+import { Value2 } from "../../../types/log";
+import { formatPrettyDecimal } from "../../../utils/format";
+import { isNumeric } from "../../../utils/type";
+import { ScoreDescriptor } from "../types";
+import styles from "./ObjectScoreDescriptor.module.css";
+
+export const objectScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
+  const buckets = values.map((val) => {
+    return JSON.stringify(val);
+  });
+  const vals = new Set(buckets);
+  let categories = undefined;
+  if (vals.size < 10) {
+    categories = Array.from(vals).map((val) => {
+      return {
+        val,
+        text: val,
+      };
+    });
+  }
+
+  return {
+    scoreType: kScoreTypeObject,
+    categories,
+    compare: () => {
+      return 0;
+    },
+    render: (score) => {
+      if (score === null || score === undefined) {
+        return "[null]";
+      }
+
+      const scores: JSX.Element[] = [];
+      const keys = Object.keys(score);
+      keys.forEach((key, index) => {
+        if (typeof score !== "object" || Array.isArray(score)) {
+          throw new Error(
+            "Unexpected us of object score descriptor for non-score object",
+          );
+        }
+        const value = score[key];
+        const formattedValue =
+          value && isNumeric(value)
+            ? formatPrettyDecimal(
+                typeof value === "number"
+                  ? value
+                  : parseFloat(value === true ? "1" : value),
+              )
+            : value;
+        scores.push(
+          <div
+            className={clsx(
+              styles.container,
+              index + 1 < keys.length ? styles.padded : undefined,
+            )}
+          >
+            <div className={clsx(styles.key, "text-size-smaller")}>{key}</div>
+            <div className={clsx(styles.value, "text-size-title")}>
+              {formattedValue}
+            </div>
+          </div>,
+        );
+      });
+
+      return scores;
+    },
+  };
+};

inspect_ai/_view/www/src/samples/descriptor/score/OtherScoreDescriptor.tsx

@@ -0,0 +1,20 @@
+import { kScoreTypeOther } from "../../../constants";
+import { RenderedContent } from "../../../metadata/RenderedContent";
+import { ScoreDescriptor } from "../types";
+
+export const otherScoreDescriptor = (): ScoreDescriptor => {
+  return {
+    scoreType: kScoreTypeOther,
+    compare: () => {
+      return 0;
+    },
+    render: (score) => {
+      return (
+        <RenderedContent
+          id="other-score-value"
+          entry={{ name: "other-score-value", value: score }}
+        />
+      );
+    },
+  };
+};

inspect_ai/_view/www/src/samples/descriptor/score/PassFailScoreDescriptor.module.css

@@ -0,0 +1,28 @@
+.circle {
+  font-family: "Consola Regular";
+  width: 20px;
+  height: 20px;
+  display: inline-flex;
+  justify-content: center;
+  align-items: center;
+  border-radius: 50%;
+  padding-top: 1px;
+}
+
+.green {
+  background-color: var(--bs-success);
+  border-color: var(--bs-success);
+  color: var(--bs-body-bg);
+}
+
+.red {
+  background-color: var(--bs-danger);
+  border-color: var(--bs-danger);
+  color: var(--bs-body-bg);
+}
+
+.orange {
+  background-color: var(--bs-orange);
+  border-color: var(--bs-orange);
+  color: var(--bs-body-bg);
+}

inspect_ai/_view/www/src/samples/descriptor/score/PassFailScoreDescriptor.tsx

@@ -0,0 +1,81 @@
+import clsx from "clsx";
+import { kScoreTypePassFail } from "../../../constants";
+import { Value2 } from "../../../types/log";
+import { ScoreDescriptor, SelectedScore } from "../types";
+import styles from "./PassFailScoreDescriptor.module.css";
+
+export const passFailScoreDescriptor = (values: Value2[]): ScoreDescriptor => {
+  const categories = [];
+  if (values.includes("C")) {
+    categories.push({
+      val: "C",
+      text: "Correct",
+    });
+  }
+  if (values.includes("P")) {
+    categories.push({
+      val: "P",
+      text: "Partial",
+    });
+  }
+  if (values.includes("I")) {
+    categories.push({
+      val: "I",
+      text: "Incorrect",
+    });
+  }
+  if (values.includes("N")) {
+    categories.push({
+      val: "N",
+      text: "Refusal",
+    });
+  }
+  const order = ["C", "P", "I", "N"];
+
+  return {
+    scoreType: kScoreTypePassFail,
+    categories,
+    render: (score: Value2) => {
+      if (score === "C") {
+        return (
+          <span
+            className={clsx("text-size-small", styles.circle, styles.green)}
+          >
+            C
+          </span>
+        );
+      } else if (score === "I") {
+        return (
+          <span className={clsx("text-size-small", styles.circle, styles.red)}>
+            I
+          </span>
+        );
+      } else if (score === "P") {
+        return (
+          <span
+            className={clsx("text-size-small", styles.circle, styles.orange)}
+          >
+            P
+          </span>
+        );
+      } else if (score === "N") {
+        return (
+          <span className={clsx("text-size-small", styles.circle, styles.red)}>
+            N
+          </span>
+        );
+      } else {
+        return String(score);
+      }
+    },
+    compare: (a: SelectedScore, b: SelectedScore) => {
+      if (typeof a.value !== "string" || typeof b.value !== "string") {
+        throw new Error(
+          "Unexpectedly using the pass fail scorer on non-string values",
+        );
+      }
+      const sort = order.indexOf(a.value || "") - order.indexOf(b.value || "");
+      return sort;
+    },
+  };
+};