inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/plan/PlanDetailView.tsx

@@ -0,0 +1,324 @@
+import { MetaDataView } from "../metadata/MetaDataView";
+import { EvalPlan, EvalScore, EvalSpec, Params2 } from "../types/log";
+import { toTitleCase } from "../utils/format";
+import { ghCommitUrl } from "../utils/git";
+import { DatasetDetailView } from "./DatasetDetailView";
+import { ScorerDetailView } from "./ScorerDetailView";
+import { SolversDetailView } from "./SolverDetailView";
+
+import clsx from "clsx";
+import styles from "./PlanDetailView.module.css";
+
+interface PlanDetailViewProps {
+  evaluation?: EvalSpec;
+  plan?: EvalPlan;
+  scores?: EvalScore[];
+}
+
+export const PlanDetailView: React.FC<PlanDetailViewProps> = ({
+  evaluation,
+  plan,
+  scores,
+}) => {
+  if (!evaluation) {
+    return null;
+  }
+
+  // Add configuration
+  const config: Record<string, unknown> = {};
+  Object.entries(evaluation?.config || {}).forEach((entry) => {
+    const key = entry[0];
+    const value = entry[1];
+    config[key] = value;
+  });
+
+  const steps = plan?.steps;
+  const metadata = evaluation?.metadata;
+  const revision = evaluation?.revision;
+  const packages = evaluation?.packages;
+  const model_args = evaluation?.model_args;
+  const task_args = evaluation?.task_args;
+  const generate_config = plan?.config;
+
+  const taskInformation: Record<string, unknown> = {
+    ["Task ID"]: evaluation?.task_id,
+    ["Run ID"]: evaluation?.run_id,
+  };
+  if (revision) {
+    taskInformation[
+      `${revision.type ? `${toTitleCase(revision.type)} ` : ""}Revision`
+    ] = {
+      _html: (
+        <a href={ghCommitUrl(revision.origin, revision.commit)}>
+          {revision.commit}
+        </a>
+      ),
+    };
+  }
+  if (packages) {
+    const names = Object.keys(packages).map((key) => {
+      return `${key} ${packages[key]}`;
+    });
+
+    if (names.length === 1) {
+      taskInformation["Inspect"] = names[0];
+    } else {
+      taskInformation["Inspect"] = names;
+    }
+  }
+  if (evaluation.tags) {
+    taskInformation["Tags"] = evaluation.tags.join(", ");
+  }
+
+  if (evaluation?.model) {
+    config["model"] = evaluation.model;
+  }
+
+  if (evaluation?.model_base_url) {
+    config["model_base_url"] = evaluation.model_base_url;
+  }
+
+  if (evaluation?.sandbox) {
+    config["sandbox"] = evaluation.sandbox[0];
+    if (evaluation.sandbox[1]) {
+      config["sandbox_config"] = evaluation.sandbox[1];
+    }
+  }
+
+  const taskColumns: {
+    title: string;
+    className: string | string[];
+    contents: React.ReactNode;
+  }[] = [];
+  taskColumns.push({
+    title: "Dataset",
+    className: styles.floatingCol,
+    contents: <DatasetDetailView dataset={evaluation.dataset} />,
+  });
+
+  if (steps) {
+    taskColumns.push({
+      title: "Solvers",
+      className: styles.wideCol,
+      contents: <SolversDetailView steps={steps} />,
+    });
+  }
+
+  if (scores) {
+    const scorers = scores.reduce(
+      (accum, score) => {
+        if (!accum[score.scorer]) {
+          accum[score.scorer] = {
+            scores: [score.name],
+            params: score.params,
+          };
+        } else {
+          accum[score.scorer].scores.push(score.name);
+        }
+        return accum;
+      },
+      {} as Record<string, { scores: string[]; params: Params2 }>,
+    );
+
+    if (Object.keys(scorers).length > 0) {
+      const label = Object.keys(scorers).length === 1 ? "Scorer" : "Scorers";
+      const scorerPanels = Object.keys(scorers).map((key) => {
+        return (
+          <ScorerDetailView
+            key={key}
+            name={key}
+            scores={scorers[key].scores}
+            params={scorers[key].params as Record<string, unknown>}
+          />
+        );
+      });
+
+      taskColumns.push({
+        title: label,
+        className: styles.floatingCol,
+        contents: scorerPanels,
+      });
+    }
+  }
+
+  // Compute the column style for the remaining (either 1 or 2 columns wide)
+  const metadataColumns: {
+    title: string;
+    className: string;
+    contents: React.ReactNode;
+  }[] = [];
+  const cols = colCount(
+    metadataColumns,
+    task_args,
+    model_args,
+    config,
+    metadata,
+  );
+
+  metadataColumns.push({
+    title: "Task Information",
+    className: cols === 1 ? styles.oneCol : styles.twoCol,
+    contents: (
+      <MetaDataView
+        key={`plan-md-task`}
+        className={"text-size-small"}
+        entries={taskInformation}
+        tableOptions="sm"
+      />
+    ),
+  });
+
+  if (task_args && Object.keys(task_args).length > 0) {
+    metadataColumns.push({
+      title: "Task Args",
+      className: cols === 1 ? styles.oneCol : styles.twoCol,
+      contents: (
+        <MetaDataView
+          key={`plan-md-task-args`}
+          className={"text-size-small"}
+          entries={task_args as Record<string, unknown>}
+          tableOptions="sm"
+        />
+      ),
+    });
+  }
+  if (model_args && Object.keys(model_args).length > 0) {
+    metadataColumns.push({
+      title: "Model Args",
+      className: cols === 1 ? styles.oneCol : styles.twoCol,
+      contents: (
+        <MetaDataView
+          key={`plan-md-model-args`}
+          className={"text-size-small"}
+          entries={model_args as Record<string, unknown>}
+          tableOptions="sm"
+        />
+      ),
+    });
+  }
+
+  if (config && Object.keys(config).length > 0) {
+    metadataColumns.push({
+      title: "Configuration",
+      className: cols === 1 ? styles.oneCol : styles.twoCol,
+      contents: (
+        <MetaDataView
+          key={`plan-md-config`}
+          className={"text-size-small"}
+          entries={config}
+          tableOptions="sm"
+        />
+      ),
+    });
+  }
+
+  if (generate_config && Object.keys(generate_config).length > 0) {
+    const generate_record: Record<string, unknown> = Object.fromEntries(
+      Object.entries(generate_config),
+    );
+
+    metadataColumns.push({
+      title: "Generate Config",
+      className: cols === 1 ? styles.oneCol : styles.twoCol,
+      contents: (
+        <MetaDataView
+          key={`plan-md-generate-config`}
+          className={"text-size-small"}
+          entries={generate_record}
+          tableOptions="sm"
+        />
+      ),
+    });
+  }
+
+  if (metadata && Object.keys(metadata).length > 0) {
+    metadataColumns.push({
+      title: "Metadata",
+      className: cols === 1 ? styles.oneCol : styles.twoCol,
+      contents: (
+        <MetaDataView
+          key={`plan-md-metadata`}
+          className={"text-size-small"}
+          entries={metadata}
+          tableOptions="sm"
+        />
+      ),
+    });
+  }
+
+  return (
+    <div className={styles.container}>
+      <div
+        className={styles.grid}
+        style={{
+          gridTemplateColumns: `repeat(${taskColumns.length}, auto)`,
+        }}
+      >
+        {taskColumns.map((col) => {
+          return (
+            <PlanColumn
+              title={col.title}
+              className={col.className}
+              key={`plan-col-${col.title}`}
+            >
+              {col.contents}
+            </PlanColumn>
+          );
+        })}
+      </div>
+
+      <div className={clsx(styles.row)}>
+        {metadataColumns.map((col) => {
+          return (
+            <PlanColumn
+              title={col.title}
+              className={col.className}
+              key={`plan-col-${col.title}`}
+            >
+              {col.contents}
+            </PlanColumn>
+          );
+        })}
+      </div>
+    </div>
+  );
+};
+
+const colCount = (...other: unknown[]) => {
+  let count = 0;
+  for (const o in other) {
+    if (o && Object.keys(o).length > 0) {
+      count++;
+    }
+  }
+  return count;
+};
+
+interface PlanColumnProps {
+  title: string;
+  className: string | string[];
+  children: React.ReactNode;
+}
+
+const PlanColumn: React.FC<PlanColumnProps> = ({
+  title,
+  className,
+  children,
+}) => {
+  return (
+    <div className={clsx(className)}>
+      <div
+        className={clsx(
+          "card-subheading",
+          "text-size-small",
+          "text-style-label",
+          "text-style-secondary",
+          styles.planCol,
+        )}
+      >
+        {title}
+      </div>
+      {children}
+    </div>
+  );
+};

inspect_ai/_view/www/src/plan/ScorerDetailView.module.css

@@ -0,0 +1,3 @@
+.item {
+  margin-bottom: 0;
+}

inspect_ai/_view/www/src/plan/ScorerDetailView.tsx

@@ -0,0 +1,30 @@
+import clsx from "clsx";
+import { ApplicationIcons } from "../appearance/icons";
+import styles from "./DatasetDetailView.module.css";
+import { DetailStep } from "./DetailStep";
+
+interface ScorerDetailViewProps {
+  name: string;
+  scores: string[];
+  params: Record<string, unknown>;
+}
+
+export const ScorerDetailView: React.FC<ScorerDetailViewProps> = ({
+  name,
+  scores,
+  params,
+}) => {
+  // Merge scores into params
+  if (scores.length > 1) {
+    params["scores"] = scores;
+  }
+
+  return (
+    <DetailStep
+      icon={ApplicationIcons.scorer}
+      name={name}
+      params={params}
+      className={clsx(styles.item, "text-size-base")}
+    />
+  );
+};

inspect_ai/_view/www/src/plan/SolverDetailView.module.css

@@ -0,0 +1,15 @@
+.container {
+  display: flex;
+  flex-direction: columns;
+}
+
+.item {
+  margin-bottom: 0;
+}
+
+.separator {
+  margin-left: 0.3em;
+  margin-right: 0.3em;
+  margin-top: 0;
+  margin-bottom: -0.1em;
+}

inspect_ai/_view/www/src/plan/SolverDetailView.tsx

@@ -0,0 +1,32 @@
+import clsx from "clsx";
+import { Fragment } from "react";
+import { ApplicationIcons } from "../appearance/icons";
+import { Steps } from "../types/log";
+import { DetailStep } from "./DetailStep";
+import styles from "./SolverDetailView.module.css";
+
+interface SolversDetailView {
+  steps: Steps;
+}
+
+export const SolversDetailView: React.FC<SolversDetailView> = ({ steps }) => {
+  const separator = (
+    <div className={clsx(styles.items, "text-size-small", styles.separator)}>
+      <i className={ApplicationIcons.arrows.right} />
+    </div>
+  );
+
+  const details = steps?.map((step, index) => {
+    return (
+      <Fragment key={`solver-step-${index}`}>
+        <DetailStep
+          name={step.solver}
+          className={clsx(styles.items, "text-size-small")}
+        />
+        {index < steps.length - 1 ? separator : ""}
+      </Fragment>
+    );
+  });
+
+  return <div className={styles.container}>{details}</div>;
+};

inspect_ai/_view/www/src/samples/InlineSampleDisplay.module.css

@@ -0,0 +1,8 @@
+.container {
+  flex-direction: row;
+  width: 100%;
+}
+
+.body {
+  margin: 1em 1em 1em 1em;
+}

inspect_ai/_view/www/src/samples/InlineSampleDisplay.tsx

@@ -0,0 +1,53 @@
+import { RefObject } from "react";
+import { ErrorPanel } from "../components/ErrorPanel";
+import { ProgressBar } from "../components/ProgressBar";
+import { EvalSample } from "../types/log";
+import { SampleDisplay } from "./SampleDisplay";
+import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
+
+import styles from "./InlineSampleDisplay.module.css";
+
+interface InlineSampleDisplayProps {
+  id: string;
+  sampleStatus: string;
+  sampleError?: Error;
+  sample?: EvalSample;
+  sampleDescriptor: SamplesDescriptor;
+  selectedTab?: string;
+  setSelectedTab: (tab: string) => void;
+  scrollRef: RefObject<HTMLDivElement | null>;
+}
+
+/**
+ * Inline Sample Display
+ */
+export const InlineSampleDisplay: React.FC<InlineSampleDisplayProps> = ({
+  id,
+  sample,
+  sampleStatus,
+  sampleError,
+  sampleDescriptor,
+  selectedTab,
+  setSelectedTab,
+  scrollRef,
+}) => {
+  return (
+    <div className={styles.container}>
+      <ProgressBar animating={sampleStatus === "loading"} />
+      <div className={styles.body}>
+        {sampleError ? (
+          <ErrorPanel title="Unable to load sample" error={sampleError} />
+        ) : (
+          <SampleDisplay
+            id={id}
+            sample={sample}
+            sampleDescriptor={sampleDescriptor}
+            selectedTab={selectedTab}
+            setSelectedTab={setSelectedTab}
+            scrollRef={scrollRef}
+          />
+        )}
+      </div>
+    </div>
+  );
+};

inspect_ai/_view/www/src/samples/SampleDialog.tsx

@@ -0,0 +1,122 @@
+import { ApplicationIcons } from "../appearance/icons";
+import { LargeModal, ModalTool, ModalTools } from "../components/LargeModal";
+
+import { Ref, RefObject, useCallback, useMemo, useRef } from "react";
+import { ErrorPanel } from "../components/ErrorPanel";
+import { EvalSample } from "../types/log";
+import { SampleDisplay } from "./SampleDisplay";
+import { SamplesDescriptor } from "./descriptor/samplesDescriptor";
+
+interface SampleDialogProps {
+  id: string;
+  title: string;
+  sampleStatus: string;
+  sampleError?: Error;
+  sample?: EvalSample;
+  sampleDescriptor: SamplesDescriptor;
+  selectedTab?: string;
+  setSelectedTab: (tab: string) => void;
+  showingSampleDialog: boolean;
+  setShowingSampleDialog: (showing: boolean) => void;
+  nextSample: () => void;
+  prevSample: () => void;
+  sampleScrollPositionRef: RefObject<number>;
+  setSampleScrollPosition: (position: number) => void;
+}
+
+/**
+ * Inline Sample Display
+ */
+export const SampleDialog: React.FC<SampleDialogProps> = ({
+  id,
+  title,
+  sample,
+  sampleDescriptor,
+  nextSample,
+  prevSample,
+  sampleStatus,
+  sampleError,
+  showingSampleDialog,
+  setShowingSampleDialog,
+  selectedTab,
+  setSelectedTab,
+  sampleScrollPositionRef,
+  setSampleScrollPosition,
+}) => {
+  const scrollRef: Ref<HTMLDivElement> = useRef(null);
+
+  const tools = useMemo<ModalTools>(() => {
+    const nextTool: ModalTool = {
+      label: "Next Sample",
+      icon: ApplicationIcons.next,
+      onClick: nextSample,
+      enabled: !!nextSample,
+    };
+
+    const prevTool: ModalTool = {
+      label: "Previous Sample",
+      icon: ApplicationIcons.previous,
+      onClick: prevSample,
+      enabled: !!prevSample,
+    };
+
+    return {
+      left: [prevTool],
+      right: [nextTool],
+    };
+  }, [prevSample, nextSample]);
+
+  const handleKeyUp = useCallback(
+    (e: KeyboardEvent) => {
+      switch (e.key) {
+        case "ArrowRight":
+          if (nextSample) {
+            nextSample();
+          }
+          break;
+        case "ArrowLeft":
+          if (prevSample) {
+            prevSample();
+          }
+          break;
+        case "Escape":
+          setShowingSampleDialog(false);
+          break;
+      }
+    },
+    [prevSample, nextSample],
+  );
+
+  const onHide = useCallback(() => {
+    setShowingSampleDialog(false);
+  }, [setShowingSampleDialog]);
+
+  // Provide the dialog
+  return (
+    <LargeModal
+      id={id}
+      detail={title}
+      detailTools={tools}
+      onkeyup={handleKeyUp}
+      visible={showingSampleDialog}
+      onHide={onHide}
+      showProgress={sampleStatus === "loading"}
+      initialScrollPositionRef={sampleScrollPositionRef}
+      setInitialScrollPosition={setSampleScrollPosition}
+      scrollRef={scrollRef}
+    >
+      {sampleError ? (
+        <ErrorPanel title="Sample Error" error={sampleError} />
+      ) : (
+        <SampleDisplay
+          id={id}
+          sample={sample}
+          sampleDescriptor={sampleDescriptor}
+          selectedTab={selectedTab}
+          setSelectedTab={setSelectedTab}
+          scrollRef={scrollRef}
+        />
+      )}
+    </LargeModal>
+  );
+};

inspect_ai/_view/www/src/samples/SampleDisplay.module.css

@@ -0,0 +1,29 @@
+.tabPanel {
+  padding-bottom: 1em;
+}
+
+.fullWidth {
+  width: 100%;
+}
+
+.metadataPanel {
+  display: flex;
+  flex-wrap: wrap;
+  align-items: flex-start;
+  gap: 1em;
+  padding-left: 0;
+  margin-top: 0.5em;
+}
+
+.padded {
+  padding-left: 0.8em;
+  margin-top: 0.4em;
+}
+
+.ansi {
+  margin: 1em 0;
+}
+
+.noTop {
+  margin-top: 0;
+}