inspect-ai 0.3.87__py3-none-any.whl → 0.3.89__py3-none-any.whl

inspect_ai/_view/www/log-schema.json

@@ -1342,6 +1342,43 @@
       "type": "object",
       "additionalProperties": false
     },
+    "EvalModelConfig": {
+      "description": "Model config.",
+      "properties": {
+        "model": {
+          "title": "Model",
+          "type": "string"
+        },
+        "config": {
+          "$ref": "#/$defs/GenerateConfig"
+        },
+        "base_url": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Base Url"
+        },
+        "args": {
+          "title": "Args",
+          "type": "object"
+        }
+      },
+      "required": [
+        "model",
+        "config",
+        "base_url",
+        "args"
+      ],
+      "title": "EvalModelConfig",
+      "type": "object",
+      "additionalProperties": false
+    },
     "EvalPlan": {
       "description": "Plan (solvers) used in evaluation.",
       "properties": {
@@ -2250,6 +2287,9 @@
           "title": "Model",
           "type": "string"
         },
+        "model_generate_config": {
+          "$ref": "#/$defs/GenerateConfig"
+        },
         "model_base_url": {
           "anyOf": [
             {
@@ -2266,6 +2306,21 @@
           "title": "Model Args",
           "type": "object"
         },
+        "model_roles": {
+          "anyOf": [
+            {
+              "additionalProperties": {
+                "$ref": "#/$defs/EvalModelConfig"
+              },
+              "type": "object"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Model Roles"
+        },
         "config": {
           "$ref": "#/$defs/EvalConfig"
         },
@@ -2355,8 +2410,10 @@
         "dataset",
         "sandbox",
         "model",
+        "model_generate_config",
         "model_base_url",
         "model_args",
+        "model_roles",
         "config",
         "revision",
         "packages",
@@ -3306,6 +3363,18 @@
           "title": "Model",
           "type": "string"
         },
+        "role": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Role"
+        },
         "input": {
           "items": {
             "anyOf": [
@@ -3426,6 +3495,7 @@
         "pending",
         "event",
         "model",
+        "role",
         "input",
         "tools",
         "tool_choice",
@@ -4595,6 +4665,18 @@
             }
           ],
           "default": null
+        },
+        "type": {
+          "anyOf": [
+            {
+              "type": "string"
+            },
+            {
+              "type": "null"
+            }
+          ],
+          "default": null,
+          "title": "Type"
         }
       },
       "required": [
@@ -4603,7 +4685,8 @@
         "arguments",
         "internal",
         "parse_error",
-        "view"
+        "view",
+        "type"
       ],
       "title": "ToolCall",
       "type": "object",

inspect_ai/_view/www/src/metadata/MetaDataView.module.css

@@ -9,7 +9,7 @@
 }
 
 .cell {
-  padding: 0.3em 0.3em 0.3em 0em;
+  padding: 0em 0.5em 0.3em 0em !important;
 }
 
 .compact .cell {

inspect_ai/_view/www/src/metadata/MetaDataView.tsx

@@ -6,7 +6,7 @@ import { RenderedContent } from "./RenderedContent";
 interface MetadataViewProps {
   id?: string;
   style?: CSSProperties;
-  entries: Record<string, unknown>;
+  entries: Record<string, unknown> | Array<{ name: string; value: unknown }>;
   tableOptions?: string;
   compact?: boolean;
   className?: string | string[];
@@ -66,11 +66,6 @@ export const MetaDataView: FC<MetadataViewProps> = ({
       )}
       style={style}
     >
-      <thead>
-        <tr>
-          <th colSpan={2} className={"th"}></th>
-        </tr>
-      </thead>
       <tbody>{entryEls}</tbody>
     </table>
   );
@@ -80,11 +75,21 @@ export const MetaDataView: FC<MetadataViewProps> = ({
 // or an array of record with name/value on way in
 // but coerce to array of records for order
 const toNameValues = (
-  entries?: Array<{ name: string; value: unknown }> | Record<string, unknown>,
+  entries?:
+    | Array<{ name: string; value: unknown }>
+    | Record<string, unknown>
+    | Array<unknown>,
 ): Array<{ name: string; value: unknown }> | undefined => {
   if (entries) {
     if (Array.isArray(entries)) {
-      return entries;
+      // filter arrays that don't contain the expected name value pairs
+      const filtered = entries.filter((entry) => {
+        if (entry && typeof entry === "object") {
+          return "name" in entry && "value" in entry;
+        }
+        return false;
+      });
+      return filtered as Array<{ name: string; value: unknown }>;
     } else {
       return Object.entries(entries || {}).map(([key, value]) => {
         return { name: key, value };

inspect_ai/_view/www/src/metadata/RenderedContent.tsx

@@ -147,6 +147,9 @@ const contentRenderers: Record<string, ContentRenderer> = {
     canRender: (entry) => {
       const isArray = Array.isArray(entry.value);
       if (isArray) {
+        if (entry.value.length === 0 || entry.value.length === 1) {
+          return true;
+        }
         const types = new Set(
           entry.value
             .filter((e: unknown) => e !== null)

inspect_ai/_view/www/src/plan/ModelCard.module.css

@@ -0,0 +1,16 @@
+.container {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(300px, 1fr));
+  row-gap: 2em;
+  column-gap: 1em;
+}
+
+.modelInfo {
+  display: grid;
+  grid-template-columns: max-content auto;
+  column-gap: 1em;
+}
+
+.role {
+  grid-column: -1/1;
+}

inspect_ai/_view/www/src/plan/ModelCard.tsx

@@ -0,0 +1,93 @@
+import { FC } from "react";
+import { ApplicationIcons } from "../appearance/icons";
+import { Card, CardBody, CardHeader } from "../components/Card";
+import { EvalModelConfig, EvalSpec } from "../types/log";
+
+import clsx from "clsx";
+import { MetaDataGrid } from "../metadata/MetaDataGrid";
+import styles from "./ModelCard.module.css";
+
+interface ModelCardProps {
+  evalSpec?: EvalSpec;
+}
+
+/**
+ * Renders the plan card
+ */
+export const ModelCard: FC<ModelCardProps> = ({ evalSpec }) => {
+  if (!evalSpec) {
+    return undefined;
+  }
+
+  const modelsInfo: Record<string, EvalModelConfig> = {
+    eval: {
+      model: evalSpec.model,
+      base_url: evalSpec.model_base_url,
+      config: evalSpec.model_generate_config,
+      args: evalSpec.model_args,
+    },
+    ...evalSpec.model_roles,
+  };
+
+  const noneEl = <span className="text-style-secondary">None</span>;
+
+  return (
+    <Card>
+      <CardHeader icon={ApplicationIcons.model} label="Models" />
+      <CardBody id={"task-model-card-body"}>
+        <div className={styles.container}>
+          {Object.keys(modelsInfo || {}).map((modelKey) => {
+            const modelInfo = modelsInfo[modelKey];
+            return (
+              <div
+                key={modelKey}
+                className={clsx(styles.modelInfo, "text-size-small")}
+              >
+                <div
+                  className={clsx(
+                    styles.role,
+                    "text-style-label",
+                    "text-style-secondary",
+                  )}
+                >
+                  {modelKey}
+                </div>
+
+                <div className={clsx("text-style-label")}>Model</div>
+                <div>{modelInfo.model}</div>
+
+                <div className={clsx("text-style-label")}>Base Url</div>
+                <div className="text-size-small">
+                  {modelInfo.base_url || noneEl}
+                </div>
+                <div className={clsx("text-style-label")}>Configuration</div>
+                <div className="text-size-small">
+                  {modelInfo.config &&
+                  Object.keys(modelInfo.config).length > 0 ? (
+                    <MetaDataGrid
+                      entries={
+                        modelInfo.config as any as Record<string, unknown>
+                      }
+                    />
+                  ) : (
+                    noneEl
+                  )}
+                </div>
+                <div className={clsx("text-style-label")}>Args</div>
+                <div className="text-size-small">
+                  {Object.keys(modelInfo.args).length > 0 ? (
+                    <MetaDataGrid
+                      entries={modelInfo.args as any as Record<string, unknown>}
+                    />
+                  ) : (
+                    noneEl
+                  )}
+                </div>
+              </div>
+            );
+          })}
+        </div>
+      </CardBody>
+    </Card>
+  );
+};

inspect_ai/_view/www/src/samples/transcript/ModelEventView.tsx

@@ -63,11 +63,15 @@ export const ModelEventView: FC<ModelEventViewProps> = ({
     }
   }
 
+  const panelTitle = event.role
+    ? `Model Call (${event.role}): ${event.model}`
+    : `Model Call: ${event.model}`;
+
   return (
     <EventPanel
       id={id}
       className={className}
-      title={formatTitle(`Model Call: ${event.model}`, totalUsage, callTime)}
+      title={formatTitle(panelTitle, totalUsage, callTime)}
       subTitle={formatTiming(event.timestamp, event.working_start)}
       icon={ApplicationIcons.model}
     >

inspect_ai/_view/www/src/samples/transcript/SampleLimitEventView.tsx

@@ -1,6 +1,6 @@
 import { FC } from "react";
 import { ApplicationIcons } from "../../appearance/icons";
-import { SampleLimitEvent, Type9 } from "../../types/log";
+import { SampleLimitEvent, Type10 } from "../../types/log";
 import { EventPanel } from "./event/EventPanel";
 
 interface SampleLimitEventViewProps {
@@ -17,7 +17,7 @@ export const SampleLimitEventView: FC<SampleLimitEventViewProps> = ({
   event,
   className,
 }) => {
-  const resolve_title = (type: Type9) => {
+  const resolve_title = (type: Type10) => {
     switch (type) {
       case "custom":
         return "Custom Limit Exceeded";
@@ -34,7 +34,7 @@ export const SampleLimitEventView: FC<SampleLimitEventViewProps> = ({
     }
   };
 
-  const resolve_icon = (type: Type9) => {
+  const resolve_icon = (type: Type10) => {
     switch (type) {
       case "custom":
         return ApplicationIcons.limits.custom;

inspect_ai/_view/www/src/samples/transcript/state/StateEventView.tsx

@@ -276,44 +276,21 @@ function setPath(
   value: unknown,
 ): void {
   const keys = parsePath(path);
-  let current: Record<string, unknown> | unknown[] = target;
+  let current: Record<string, unknown> = target;
 
   for (let i = 0; i < keys.length - 1; i++) {
     const key = keys[i];
-
-    if (Array.isArray(current)) {
-      const numericIndex = getIndex(key);
-      current[numericIndex] = isArrayIndex(keys[i + 1]) ? [] : {};
-      current = current[numericIndex] as
-        | Record<string, unknown>
-        | Array<unknown>;
-    } else {
-      if (!(key in current)) {
-        // If the next key is a number, create an array, otherwise an object
-        current[key] = isArrayIndex(keys[i + 1]) ? [] : {};
-      }
-      current = current[key] as Record<string, unknown> | Array<unknown>;
+    if (!(key in current)) {
+      // If the next key is a number, create an array, otherwise an object
+      current[key] = isArrayIndex(keys[i + 1]) ? [] : {};
     }
+    current = current[key] as Record<string, unknown>;
   }
 
   const lastKey = keys[keys.length - 1];
-  if (Array.isArray(current)) {
-    const numericIndex = getIndex(lastKey);
-    current[numericIndex] = value;
-  } else {
-    current[lastKey] = value;
-  }
+  current[lastKey] = value;
 }
 
-const getIndex = (key: string): number => {
-  const numericIndex = isArrayIndex(key) ? parseInt(key) : undefined;
-  if (numericIndex === undefined) {
-    throw new Error(`The key ${key} isn't a valid Array index!`);
-  }
-
-  return numericIndex;
-};
-
 /**
  * Places structure in an object (without placing values)
  */