inspect-ai 0.3.92__py3-none-any.whl → 0.3.94__py3-none-any.whl

inspect_ai/_view/www/src/@types/log.d.ts

@@ -263,9 +263,10 @@ export type Type9 =
   | "permission"
   | "file_not_found"
   | "is_a_directory"
-  | "output_limit"
+  | "limit"
   | "approval"
-  | "unknown";
+  | "unknown"
+  | "output_limit";
 export type Message1 = string;
 export type Choices = string[] | null;
 export type Target = string | string[];
@@ -315,6 +316,7 @@ export type Explanation = string | null;
 export type Metadata6 = {
   [k: string]: unknown;
 } | null;
+export type SpanId = string | null;
 export type Timestamp = string;
 export type WorkingStart = number;
 export type Pending = boolean | null;
@@ -338,6 +340,7 @@ export type Files1 = {
 } | null;
 export type Setup1 = string | null;
 export type JsonValue = unknown;
+export type SpanId1 = string | null;
 export type Timestamp1 = string;
 export type WorkingStart1 = number;
 export type Pending1 = boolean | null;
@@ -351,6 +354,7 @@ export type Type10 =
   | "custom";
 export type Message2 = string;
 export type Limit1 = number | null;
+export type SpanId2 = string | null;
 export type Timestamp2 = string;
 export type WorkingStart2 = number;
 export type Pending2 = boolean | null;
@@ -365,6 +369,7 @@ export type Input2 = string | null;
 export type Result = number | null;
 export type Output = string | null;
 export type Completed = string | null;
+export type SpanId3 = string | null;
 export type Timestamp3 = string;
 export type WorkingStart3 = number;
 export type Pending3 = boolean | null;
@@ -373,11 +378,13 @@ export type Op = "remove" | "add" | "replace" | "move" | "test" | "copy";
 export type Path = string;
 export type From = string | null;
 export type Changes = JsonChange[];
+export type SpanId4 = string | null;
 export type Timestamp4 = string;
 export type WorkingStart4 = number;
 export type Pending4 = boolean | null;
 export type Event4 = "store";
 export type Changes1 = JsonChange[];
+export type SpanId5 = string | null;
 export type Timestamp5 = string;
 export type WorkingStart5 = number;
 export type Pending5 = boolean | null;
@@ -398,11 +405,13 @@ export type Additionalproperties1 = boolean;
 export type Tools1 = ToolInfo[];
 export type ToolChoice = ("auto" | "any" | "none") | ToolFunction;
 export type Name9 = string;
+export type Retries = number | null;
 export type Error1 = string | null;
 export type Cache = ("read" | "write") | null;
 export type Time1 = number | null;
 export type Completed1 = string | null;
 export type WorkingTime = number | null;
+export type SpanId6 = string | null;
 export type Timestamp6 = string;
 export type WorkingStart6 = number;
 export type Pending6 = boolean | null;
@@ -427,6 +436,7 @@ export type Result1 =
       | ContentVideo
     )[];
 export type Truncated = [unknown, unknown] | null;
+export type SpanId7 = string | null;
 export type Timestamp7 = string;
 export type WorkingStart7 = number;
 export type Pending7 = boolean | null;
@@ -440,22 +450,26 @@ export type Decision =
   | "escalate"
   | "terminate";
 export type Explanation1 = string | null;
+export type SpanId8 = string | null;
 export type Timestamp8 = string;
 export type WorkingStart8 = number;
 export type Pending8 = boolean | null;
 export type Event8 = "input";
 export type Input4 = string;
 export type InputAnsi = string;
+export type SpanId9 = string | null;
 export type Timestamp9 = string;
 export type WorkingStart9 = number;
 export type Pending9 = boolean | null;
 export type Event9 = "score";
 export type Target2 = string | string[] | null;
 export type Intermediate = boolean;
+export type SpanId10 = string | null;
 export type Timestamp10 = string;
 export type WorkingStart10 = number;
 export type Pending10 = boolean | null;
 export type Event10 = "error";
+export type SpanId11 = string | null;
 export type Timestamp11 = string;
 export type WorkingStart11 = number;
 export type Pending11 = boolean | null;
@@ -475,24 +489,42 @@ export type Created1 = number;
 export type Filename = string;
 export type Module = string;
 export type Lineno = number;
+export type SpanId12 = string | null;
 export type Timestamp12 = string;
 export type WorkingStart12 = number;
 export type Pending12 = boolean | null;
 export type Event12 = "info";
 export type Source4 = string | null;
+export type SpanId13 = string | null;
 export type Timestamp13 = string;
 export type WorkingStart13 = number;
 export type Pending13 = boolean | null;
-export type Event13 = "step";
-export type Action1 = "begin" | "end";
+export type Event13 = "span_begin";
+export type Id8 = string;
+export type ParentId = string | null;
 export type Type13 = string | null;
 export type Name11 = string;
+export type SpanId14 = string | null;
 export type Timestamp14 = string;
 export type WorkingStart14 = number;
 export type Pending14 = boolean | null;
-export type Event14 = "subtask";
-export type Name12 = string;
+export type Event14 = "span_end";
+export type Id9 = string;
+export type SpanId15 = string | null;
+export type Timestamp15 = string;
+export type WorkingStart15 = number;
+export type Pending15 = boolean | null;
+export type Event15 = "step";
+export type Action1 = "begin" | "end";
 export type Type14 = string | null;
+export type Name12 = string;
+export type SpanId16 = string | null;
+export type Timestamp16 = string;
+export type WorkingStart16 = number;
+export type Pending16 = boolean | null;
+export type Event16 = "subtask";
+export type Name13 = string;
+export type Type15 = string | null;
 export type Events2 = (
   | SampleInitEvent
   | SampleLimitEvent
@@ -507,6 +539,8 @@ export type Events2 = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -526,6 +560,8 @@ export type Events1 = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -547,6 +583,8 @@ export type Events = (
   | ErrorEvent
   | LoggerEvent
   | InfoEvent
+  | SpanBeginEvent
+  | SpanEndEvent
   | StepEvent
   | SubtaskEvent
 )[];
@@ -554,7 +592,7 @@ export type TotalTime = number | null;
 export type WorkingTime3 = number | null;
 export type Uuid = string | null;
 export type ErrorRetries = EvalError[] | null;
-export type Type15 =
+export type Type16 =
   | "context"
   | "time"
   | "working"
@@ -1120,6 +1158,7 @@ export interface Store {
  * Beginning of processing a Sample.
  */
 export interface SampleInitEvent {
+  span_id: SpanId;
   timestamp: Timestamp;
   working_start: WorkingStart;
   pending: Pending;
@@ -1144,6 +1183,7 @@ export interface Sample {
  * The sample was unable to finish processing due to a limit
  */
 export interface SampleLimitEvent {
+  span_id: SpanId1;
   timestamp: Timestamp1;
   working_start: WorkingStart1;
   pending: Pending1;
@@ -1156,6 +1196,7 @@ export interface SampleLimitEvent {
  * Sandbox execution or I/O
  */
 export interface SandboxEvent {
+  span_id: SpanId2;
   timestamp: Timestamp2;
   working_start: WorkingStart2;
   pending: Pending2;
@@ -1173,6 +1214,7 @@ export interface SandboxEvent {
  * Change to the current `TaskState`
  */
 export interface StateEvent {
+  span_id: SpanId3;
   timestamp: Timestamp3;
   working_start: WorkingStart3;
   pending: Pending3;
@@ -1197,6 +1239,7 @@ export interface JsonChange {
  * Change to data within the current `Store`.
  */
 export interface StoreEvent {
+  span_id: SpanId4;
   timestamp: Timestamp4;
   working_start: WorkingStart4;
   pending: Pending4;
@@ -1207,6 +1250,7 @@ export interface StoreEvent {
  * Call to a language model.
  */
 export interface ModelEvent {
+  span_id: SpanId5;
   timestamp: Timestamp5;
   working_start: WorkingStart5;
   pending: Pending5;
@@ -1218,6 +1262,7 @@ export interface ModelEvent {
   tool_choice: ToolChoice;
   config: GenerateConfig;
   output: ModelOutput;
+  retries: Retries;
   error: Error1;
   cache: Cache;
   call: ModelCall | null;
@@ -1288,6 +1333,7 @@ export interface Response {
  * Call to a tool.
  */
 export interface ToolEvent {
+  span_id: SpanId6;
   timestamp: Timestamp6;
   working_start: WorkingStart6;
   pending: Pending6;
@@ -1314,6 +1360,7 @@ export interface Arguments1 {
  * Tool approval.
  */
 export interface ApprovalEvent {
+  span_id: SpanId7;
   timestamp: Timestamp7;
   working_start: WorkingStart7;
   pending: Pending7;
@@ -1340,6 +1387,7 @@ export interface ToolCallView {
  * Input screen interaction.
  */
 export interface InputEvent {
+  span_id: SpanId8;
   timestamp: Timestamp8;
   working_start: WorkingStart8;
   pending: Pending8;
@@ -1354,6 +1402,7 @@ export interface InputEvent {
  * resulting from a call to `score`.
  */
 export interface ScoreEvent {
+  span_id: SpanId9;
   timestamp: Timestamp9;
   working_start: WorkingStart9;
   pending: Pending9;
@@ -1366,6 +1415,7 @@ export interface ScoreEvent {
  * Event with sample error.
  */
 export interface ErrorEvent {
+  span_id: SpanId10;
   timestamp: Timestamp10;
   working_start: WorkingStart10;
   pending: Pending10;
@@ -1376,6 +1426,7 @@ export interface ErrorEvent {
  * Log message recorded with Python logger.
  */
 export interface LoggerEvent {
+  span_id: SpanId11;
   timestamp: Timestamp11;
   working_start: WorkingStart11;
   pending: Pending11;
@@ -1398,6 +1449,7 @@ export interface LoggingMessage {
  * Event with custom info/data.
  */
 export interface InfoEvent {
+  span_id: SpanId12;
   timestamp: Timestamp12;
   working_start: WorkingStart12;
   pending: Pending12;
@@ -1406,27 +1458,54 @@ export interface InfoEvent {
   data: JsonValue;
 }
 /**
- * Step within current sample or subtask.
+ * Mark the beginning of a transcript span.
  */
-export interface StepEvent {
+export interface SpanBeginEvent {
+  span_id: SpanId13;
   timestamp: Timestamp13;
   working_start: WorkingStart13;
   pending: Pending13;
   event: Event13;
-  action: Action1;
+  id: Id8;
+  parent_id: ParentId;
   type: Type13;
   name: Name11;
 }
 /**
- * Subtask spawned.
+ * Mark the end of a transcript span.
  */
-export interface SubtaskEvent {
+export interface SpanEndEvent {
+  span_id: SpanId14;
   timestamp: Timestamp14;
   working_start: WorkingStart14;
   pending: Pending14;
   event: Event14;
-  name: Name12;
+  id: Id9;
+}
+/**
+ * Step within current sample or subtask.
+ */
+export interface StepEvent {
+  span_id: SpanId15;
+  timestamp: Timestamp15;
+  working_start: WorkingStart15;
+  pending: Pending15;
+  event: Event15;
+  action: Action1;
   type: Type14;
+  name: Name12;
+}
+/**
+ * Subtask spawned.
+ */
+export interface SubtaskEvent {
+  span_id: SpanId16;
+  timestamp: Timestamp16;
+  working_start: WorkingStart16;
+  pending: Pending16;
+  event: Event16;
+  name: Name13;
+  type: Type15;
   input: Input5;
   result: Result2;
   events: Events2;
@@ -1449,7 +1528,7 @@ export interface Attachments {
  * Limit encontered by sample.
  */
 export interface EvalSampleLimit {
-  type: Type15;
+  type: Type16;
   limit: Limit2;
 }
 /**

inspect_ai/_view/www/src/app/content/MetaDataGrid.tsx

@@ -41,13 +41,13 @@ export const MetaDataGrid: FC<MetadataGridProps> = ({
             styles.cell,
             "text-style-label",
             "text-style-secondary",
-            "text-size-small",
+            "text-size-smaller",
           )}
         >
           {entry.name}
         </div>
         <div
-          className={clsx(styles.value, `${baseId}-value`, "text-size-small")}
+          className={clsx(styles.value, `${baseId}-value`, "text-size-smaller")}
         >
           <RenderedContent id={id} entry={entry} />
         </div>

inspect_ai/_view/www/src/app/content/MetaDataView.module.css

@@ -9,7 +9,7 @@
 }
 
 .cell {
-  padding: 0em 0.5em 0.3em 0em !important;
+  padding: 0em 0.7em 0.3em 0em !important;
 }
 
 .compact .cell {

inspect_ai/_view/www/src/app/content/MetadataGrid.module.css

@@ -1,7 +1,7 @@
 .grid {
   display: grid;
   grid-template-columns: max-content auto;
-  column-gap: 1em;
+  column-gap: 0.5em;
   row-gap: 0.2em;
 }
 

inspect_ai/_view/www/src/app/content/RenderedContent.tsx

@@ -54,7 +54,7 @@ export const RenderedContent: FC<RenderedContentProps> = ({
       if (typeof entry.value === "object") {
         return JSON.stringify(entry.value);
       }
-      return String(entry.value);
+      return String(entry.value).trim();
     } catch (e) {
       return "[Unable to display value]";
     }

inspect_ai/_view/www/src/app/log-view/LogView.tsx

@@ -17,7 +17,9 @@ import { useLogNavigation } from "../routing/navigationHooks";
 import styles from "./LogView.module.css";
 import { useInfoTabConfig } from "./tabs/InfoTab";
 import { useJsonTabConfig } from "./tabs/JsonTab";
+import { useModelsTab } from "./tabs/ModelsTab";
 import { useSamplesTabConfig } from "./tabs/SamplesTab";
+import { useTaskTabConfig } from "./tabs/TaskTab";
 import { TabDescriptor } from "./types";
 
 export const LogView: FC = () => {
@@ -45,7 +47,14 @@ export const LogView: FC = () => {
     selectedLogSummary?.plan,
     selectedLogSummary?.error,
     selectedLogSummary?.results,
+  );
+
+  const taskTabConfig = useTaskTabConfig(evalSpec, selectedLogSummary?.stats);
+
+  const modelsTabConfig = useModelsTab(
+    evalSpec,
     selectedLogSummary?.stats,
+    selectedLogSummary?.status,
   );
 
   const jsonTabConfig = useJsonTabConfig(
@@ -60,6 +69,8 @@ export const LogView: FC = () => {
 
   const tabs: Record<string, TabDescriptor<any>> = {
     ...(samplesTabConfig ? { samples: samplesTabConfig } : {}),
+    task: taskTabConfig,
+    model: modelsTabConfig,
     config: configTabConfig,
     json: jsonTabConfig,
   };

inspect_ai/_view/www/src/app/log-view/tabs/InfoTab.tsx

@@ -6,14 +6,12 @@ import {
   EvalSpec,
   EvalStats,
 } from "../../../@types/log";
-import { UsageCard } from "../../usage/UsageCard";
-import { TaskErrorCard } from "../error/TaskErrorPanel";
 import { SampleSummary } from "../../../client/api/types";
 import { MessageBand } from "../../../components/MessageBand";
-import { ModelCard } from "../../plan/ModelCard";
 import { kLogViewInfoTabId } from "../../../constants";
 import { useTotalSampleCount } from "../../../state/hooks";
 import { PlanCard } from "../../plan/PlanCard";
+import { TaskErrorCard } from "../error/TaskErrorPanel";
 
 // Individual hook for Info tab
 export const useInfoTabConfig = (
@@ -21,7 +19,6 @@ export const useInfoTabConfig = (
   evalPlan: EvalPlan | undefined,
   evalError: EvalError | undefined | null,
   evalResults: EvalResults | undefined | null,
-  evalStats: EvalStats | undefined,
 ) => {
   const totalSampleCount = useTotalSampleCount();
   return useMemo(() => {
@@ -35,11 +32,10 @@ export const useInfoTabConfig = (
         evalPlan,
         evalError,
         evalResults,
-        evalStats,
         sampleCount: totalSampleCount,
       },
     };
-  }, [evalSpec, evalPlan, evalError, evalResults, evalStats, totalSampleCount]);
+  }, [evalSpec, evalPlan, evalError, evalResults, totalSampleCount]);
 };
 
 interface PlanTabProps {
@@ -57,7 +53,6 @@ export const InfoTab: FC<PlanTabProps> = ({
   evalSpec,
   evalPlan,
   evalResults,
-  evalStats,
   evalStatus,
   evalError,
   sampleCount,
@@ -85,8 +80,6 @@ export const InfoTab: FC<PlanTabProps> = ({
           evalPlan={evalPlan}
           scores={evalResults?.scores}
         />
-        {evalSpec ? <ModelCard evalSpec={evalSpec} /> : undefined}
-        {evalStatus !== "started" ? <UsageCard stats={evalStats} /> : undefined}
         {evalStatus === "error" && evalError ? (
           <TaskErrorCard error={evalError} />
         ) : undefined}

inspect_ai/_view/www/src/app/log-view/tabs/ModelsTab.tsx

@@ -0,0 +1,51 @@
+import { FC, useMemo } from "react";
+import { EvalSpec, EvalStats, Status } from "../../../@types/log";
+import { kLogViewModelsTabId } from "../../../constants";
+import { ModelCard } from "../../plan/ModelCard";
+import { UsageCard } from "../../usage/UsageCard";
+
+// Individual hook for Info tab
+export const useModelsTab = (
+  evalSpec: EvalSpec | undefined,
+  evalStats: EvalStats | undefined,
+  evalStatus?: Status,
+) => {
+  return useMemo(() => {
+    return {
+      id: kLogViewModelsTabId,
+      label: "Models",
+      scrollable: true,
+      component: ModelTab,
+      componentProps: {
+        evalSpec,
+        evalStats,
+        evalStatus,
+      },
+    };
+  }, [evalSpec, evalStats]);
+};
+
+interface ModelTabProps {
+  evalSpec?: EvalSpec;
+  evalStats?: EvalStats;
+  evalStatus?: Status;
+}
+
+export const ModelTab: FC<ModelTabProps> = ({
+  evalSpec,
+  evalStats,
+  evalStatus,
+}) => {
+  return (
+    <div style={{ width: "100%" }}>
+      <div style={{ padding: "0.5em 1em 0 1em", width: "100%" }}>
+        {evalSpec ? <ModelCard evalSpec={evalSpec} /> : undefined}
+        {evalStatus !== "started" &&
+          evalStats?.model_usage &&
+          Object.keys(evalStats.model_usage).length > 0 && (
+            <UsageCard stats={evalStats} />
+          )}
+      </div>
+    </div>
+  );
+};

inspect_ai/_view/www/src/app/log-view/tabs/TaskTab.module.css

@@ -0,0 +1,6 @@
+.grid {
+  display: grid;
+  grid-template-columns: fit-content(50%) fit-content(50%);
+  column-gap: 3em;
+  align-items: flex-start;
+}

inspect_ai/_view/www/src/app/log-view/tabs/TaskTab.tsx

@@ -0,0 +1,143 @@
+import clsx from "clsx";
+import { FC, useMemo } from "react";
+import { EvalSpec, EvalStats } from "../../../@types/log";
+import { Card, CardBody, CardHeader } from "../../../components/Card";
+import { kLogViewTaskTabId } from "../../../constants";
+import { formatDuration, toTitleCase } from "../../../utils/format";
+import { ghCommitUrl } from "../../../utils/git";
+import { MetaDataView } from "../../content/MetaDataView";
+
+import styles from "./TaskTab.module.css";
+
+// Individual hook for Info tab
+export const useTaskTabConfig = (
+  evalSpec: EvalSpec | undefined,
+  evalStats?: EvalStats,
+) => {
+  return useMemo(() => {
+    return {
+      id: kLogViewTaskTabId,
+      label: "Task",
+      scrollable: true,
+      component: TaskTab,
+      componentProps: {
+        evalSpec,
+        evalStats,
+      },
+    };
+  }, [evalSpec, evalStats]);
+};
+
+interface TaskTabProps {
+  evalSpec?: EvalSpec;
+  evalStats?: EvalStats;
+}
+
+export const TaskTab: FC<TaskTabProps> = ({ evalSpec, evalStats }) => {
+  const config: Record<string, unknown> = {};
+  Object.entries(evalSpec?.config || {}).forEach((entry) => {
+    const key = entry[0];
+    const value = entry[1];
+    config[key] = value;
+  });
+
+  const revision = evalSpec?.revision;
+  const packages = evalSpec?.packages;
+
+  const taskInformation: Record<string, unknown> = {
+    ["Task ID"]: evalSpec?.task_id,
+    ["Run ID"]: evalSpec?.run_id,
+  };
+
+  if (revision) {
+    taskInformation[
+      `${revision.type ? `${toTitleCase(revision.type)} ` : ""}Revision`
+    ] = {
+      _html: (
+        <a href={ghCommitUrl(revision.origin, revision.commit)}>
+          {revision.commit}
+        </a>
+      ),
+    };
+  }
+  if (packages) {
+    const names = Object.keys(packages).map((key) => {
+      return `${key} ${packages[key]}`;
+    });
+
+    if (names.length === 1) {
+      taskInformation["Inspect"] = names[0];
+    } else {
+      taskInformation["Inspect"] = names;
+    }
+  }
+  if (evalSpec?.tags) {
+    taskInformation["tags"] = evalSpec?.tags.join(", ");
+  }
+
+  if (evalSpec?.sandbox) {
+    if (Array.isArray(evalSpec?.sandbox)) {
+      taskInformation["sandbox"] = evalSpec.sandbox[0];
+      if (evalSpec.sandbox[1]) {
+        taskInformation["sandbox_config"] = evalSpec.sandbox[1];
+      }
+    } else {
+      taskInformation["sandbox"] = evalSpec?.sandbox.type;
+      taskInformation["sandbox_config"] = evalSpec?.sandbox.config;
+    }
+  }
+
+  const totalDuration = formatDuration(
+    new Date(evalStats?.started_at || 0),
+    new Date(evalStats?.completed_at || 0),
+  );
+
+  const task_args = evalSpec?.task_args || {};
+
+  return (
+    <div style={{ width: "100%" }}>
+      <div style={{ padding: "0.5em 1em 0 1em", width: "100%" }}>
+        <Card>
+          <CardHeader label="Task Info" />
+          <CardBody id={"task-card-config"}>
+            <div className={clsx(styles.grid)}>
+              <MetaDataView
+                key={`plan-md-task`}
+                className={"text-size-small"}
+                entries={taskInformation}
+                tableOptions="sm"
+              />
+
+              <MetaDataView
+                entries={{
+                  ["Start"]: new Date(
+                    evalStats?.started_at || 0,
+                  ).toLocaleString(),
+                  ["End"]: new Date(
+                    evalStats?.completed_at || 0,
+                  ).toLocaleString(),
+                  ["Duration"]: totalDuration,
+                }}
+                tableOptions="sm"
+              />
+            </div>
+          </CardBody>
+        </Card>
+
+        {Object.keys(task_args).length > 0 && (
+          <Card>
+            <CardHeader label="Task Args" />
+            <CardBody id={"task-card-config"}>
+              <MetaDataView
+                key={`plan-md-task-args`}
+                className={"text-size-small"}
+                entries={task_args as Record<string, unknown>}
+                tableOptions="sm"
+              />
+            </CardBody>
+          </Card>
+        )}
+      </div>
+    </div>
+  );
+};