inspect-ai 0.3.62__py3-none-any.whl → 0.3.64__py3-none-any.whl

inspect_ai/_view/www/src/utils/queue.ts

@@ -0,0 +1,51 @@
+/**
+ * A class that manages asynchronous task execution with a limit on concurrent tasks.
+ */
+export class AsyncQueue {
+  constructor(concurrentLimit: number = 6) {
+    this.concurrentLimit = concurrentLimit;
+    this.queue = [];
+    this.runningCount = 0;
+  }
+
+  // Max concurrency
+  private readonly concurrentLimit: number;
+
+  // The queue
+  private readonly queue: Array<() => Promise<void>>;
+
+  // Count of currently running tasks
+  private runningCount: number;
+
+  // Adds a task to the queue and runs it if the concurrency limit allows.
+  async enqueue<T>(task: () => Promise<T>): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      this.queue.push(async () => {
+        try {
+          const result = await task();
+          resolve(result);
+        } catch (error) {
+          reject(error);
+        } finally {
+          this.runningCount--;
+          this.runNext();
+        }
+      });
+
+      if (this.runningCount < this.concurrentLimit) {
+        this.runNext();
+      }
+    });
+  }
+
+  // Runs the next task in the queue if there are available slots for concurrent execution.
+  private runNext(): void {
+    if (this.queue.length > 0 && this.runningCount < this.concurrentLimit) {
+      const task = this.queue.shift();
+      if (task) {
+        this.runningCount++;
+        task();
+      }
+    }
+  }
+}

inspect_ai/_view/www/src/utils/sync.ts

@@ -0,0 +1,114 @@
+/**
+ * Delays the execution of code for a specified number of milliseconds.
+ */
+export function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Creates a throttled version of a function that only invokes the original function
+ * at most once per every `wait` milliseconds. The throttled function will run as much
+ * as it can, without ever going more than once per `wait` duration.
+ */
+export function throttle<T extends (...args: any[]) => any>(
+  func: T,
+  wait: number,
+  options: { leading?: boolean; trailing?: boolean } = {},
+): (...args: Parameters<T>) => ReturnType<T> {
+  let context: any;
+  let args: Parameters<T> | null;
+  let result: ReturnType<T>;
+  let timeout: ReturnType<typeof setTimeout> | null = null;
+  let previous = 0;
+
+  const later = function (): void {
+    previous = options.leading === false ? 0 : Date.now();
+    timeout = null;
+    result = func.apply(context, args === null ? [] : args);
+    if (!timeout) {
+      context = null;
+      args = null;
+    }
+  };
+
+  return function (this: any, ...callArgs: Parameters<T>): ReturnType<T> {
+    const now = Date.now();
+    if (!previous && options.leading === false) {
+      previous = now;
+    }
+    const remaining = wait - (now - previous);
+
+    context = this;
+    args = callArgs as unknown as Parameters<T>;
+
+    if (remaining <= 0 || remaining > wait) {
+      if (timeout) {
+        clearTimeout(timeout);
+        timeout = null;
+      }
+      previous = now;
+      result = func.apply(context, args);
+      if (!timeout) {
+        context = null;
+        args = null;
+      }
+    } else if (!timeout && options.trailing !== false) {
+      timeout = setTimeout(later, remaining);
+    }
+
+    return result;
+  };
+}
+
+/**
+ * Creates a debounced version of a function that delays invoking the function
+ * until after `wait` milliseconds have passed since the last time it was invoked.
+ */
+export function debounce<T extends (...args: any[]) => any>(
+  func: T,
+  wait: number,
+  options: { leading?: boolean; trailing?: boolean } = {},
+): (...args: Parameters<T>) => ReturnType<T> {
+  let timeout: ReturnType<typeof setTimeout> | null = null;
+  let context: any;
+  let args: Parameters<T>;
+  let result: ReturnType<T>;
+  let lastCallTime: number | null = null;
+
+  const later = (): void => {
+    const last = Date.now() - (lastCallTime || 0);
+
+    if (last < wait && last >= 0) {
+      timeout = setTimeout(later, wait - last);
+    } else {
+      timeout = null;
+      if (!options.leading) {
+        result = func.apply(context, args);
+        if (!timeout) {
+          context = null;
+          args = null!;
+        }
+      }
+    }
+  };
+
+  return function (this: any, ...callArgs: Parameters<T>): ReturnType<T> {
+    context = this;
+    args = callArgs;
+    lastCallTime = Date.now();
+
+    const callNow = options.leading && !timeout;
+
+    if (!timeout) {
+      timeout = setTimeout(later, wait);
+    }
+
+    if (callNow) {
+      result = func.apply(context, args);
+      context = null;
+      args = null!;
+    }
+
+    return result;
+  };
+}

inspect_ai/_view/www/src/utils/{Type.mjs → type.ts}

@@ -1,12 +1,9 @@
 // @ts-check
 /**
  * Checks if a given value is numeric.
- *
- * @param {*} n - The value to check.
- * @returns {boolean} - `true` if the value is numeric, `false` otherwise.
  */
-export const isNumeric = (n) => {
-  return !isNaN(parseFloat(n)) && isFinite(n);
+export const isNumeric = (n: unknown): boolean => {
+  return !isNaN(parseFloat(n as any)) && isFinite(n as any);
 };
 
 /**
@@ -15,7 +12,7 @@ export const isNumeric = (n) => {
  * @param {*} val - The value to ensure is an array.
  * @returns {Array} - an Array
  */
-export const toArray = (val) => {
+export const toArray = <T>(val: T | T[]): Array<T> => {
   if (Array.isArray(val)) {
     return val;
   } else {

inspect_ai/_view/www/src/utils/vscode.ts

@@ -34,3 +34,16 @@ export const getVscodeApi = (): VSCodeApi | undefined => {
     return undefined;
   }
 };
+
+// Determine the capabilities
+// If this is vscode, check for the version meta
+// so we know it supports downloads
+export const isVscode = () => {
+  const bodyEl = document.querySelector("body");
+  return (
+    bodyEl !== null &&
+    !!bodyEl.getAttributeNames().find((attr) => {
+      return attr.includes("data-vscode-");
+    })
+  );
+};

inspect_ai/_view/www/src/workspace/WorkSpace.tsx

@@ -0,0 +1,324 @@
+import { ApplicationIcons } from "../appearance/icons";
+import { ToolButton } from "../components/ToolButton";
+import { SampleTools } from "../samples/SamplesTools";
+import { JsonTab } from "./tabs/JsonTab";
+import { SamplesTab } from "./tabs/SamplesTab";
+
+import clsx from "clsx";
+import { MouseEvent, RefObject, useEffect, useMemo, useRef } from "react";
+import { SampleSummary } from "../api/types.ts";
+import {
+  kEvalWorkspaceTabId,
+  kInfoWorkspaceTabId,
+  kJsonWorkspaceTabId,
+} from "../constants";
+import { SamplesDescriptor } from "../samples/descriptor/samplesDescriptor";
+import {
+  Capabilities,
+  CurrentLog,
+  SampleMode,
+  ScoreFilter,
+  ScoreLabel,
+} from "../types.ts";
+import {
+  Epochs,
+  EvalError,
+  EvalPlan,
+  EvalResults,
+  EvalSample,
+  EvalSpec,
+  EvalStats,
+  Status,
+} from "../types/log";
+import { InfoTab } from "./tabs/InfoTab.tsx";
+import { WorkSpaceView } from "./WorkSpaceView.tsx";
+
+interface WorkSpaceProps {
+  task_id?: string;
+  logFileName?: string;
+  evalError?: EvalError;
+  evalStatus?: Status;
+  evalVersion?: number;
+  evalSpec?: EvalSpec;
+  evalPlan?: EvalPlan;
+  evalStats?: EvalStats;
+  evalResults?: EvalResults;
+  log?: CurrentLog;
+  samples?: SampleSummary[];
+  sampleMode: SampleMode;
+  groupBy: "none" | "epoch" | "sample";
+  groupByOrder: "asc" | "desc";
+  selectedSample?: EvalSample;
+  sampleStatus: string;
+  sampleError?: Error;
+  showToggle: boolean;
+  refreshLog: () => Promise<void>;
+  capabilities: Capabilities;
+  selectedSampleIndex: number;
+  samplesDescriptor?: SamplesDescriptor;
+  setSelectedSampleIndex: (index: number) => void;
+  selectedSampleTab?: string;
+  setSelectedSampleTab: (tab: string) => void;
+  sort: string;
+  setSort: (sort: string) => void;
+  epochs?: Epochs;
+  epoch: string;
+  showingSampleDialog: boolean;
+  setShowingSampleDialog: (showing: boolean) => void;
+  setEpoch: (epoch: string) => void;
+  filter: ScoreFilter;
+  setFilter: (filter: ScoreFilter) => void;
+  score?: ScoreLabel;
+  setScore: (score: ScoreLabel) => void;
+  scores: ScoreLabel[];
+  offcanvas: boolean;
+  setOffcanvas: (offcanvas: boolean) => void;
+  selectedTab: string;
+  setSelectedTab: (id: string) => void;
+  sampleScrollPositionRef: RefObject<number>;
+  setSampleScrollPosition: (position: number) => void;
+  workspaceTabScrollPositionRef: RefObject<Record<string, number>>;
+  setWorkspaceTabScrollPosition: (tab: string, position: number) => void;
+}
+
+/**
+ * Renders the Main Application
+ */
+export const WorkSpace: React.FC<WorkSpaceProps> = (props) => {
+  const {
+    task_id,
+    evalStatus,
+    logFileName,
+    evalSpec,
+    evalPlan,
+    evalStats,
+    evalResults,
+    samples,
+    showToggle,
+    offcanvas,
+    setOffcanvas,
+    samplesDescriptor,
+    selectedTab,
+    setSelectedTab,
+    workspaceTabScrollPositionRef,
+    setWorkspaceTabScrollPosition,
+  } = props;
+  if (!evalSpec) {
+    return null;
+  }
+
+  const divRef = useRef<HTMLDivElement>(null);
+
+  // Display the log
+  useEffect(() => {
+    if (divRef.current) {
+      divRef.current.scrollTop = 0;
+    }
+  }, [task_id]);
+
+  const resolvedTabs = useResolvedTabs(props);
+
+  return (
+    <WorkSpaceView
+      logFileName={logFileName}
+      divRef={divRef}
+      evalSpec={evalSpec}
+      evalPlan={evalPlan}
+      evalResults={evalResults}
+      evalStats={evalStats}
+      samples={samples}
+      evalDescriptor={samplesDescriptor?.evalDescriptor}
+      status={evalStatus}
+      tabs={resolvedTabs}
+      selectedTab={selectedTab}
+      showToggle={showToggle}
+      offcanvas={offcanvas}
+      setSelectedTab={setSelectedTab}
+      workspaceTabScrollPositionRef={workspaceTabScrollPositionRef}
+      setWorkspaceTabScrollPosition={setWorkspaceTabScrollPosition}
+      setOffcanvas={setOffcanvas}
+    />
+  );
+};
+
+const copyFeedback = (e: MouseEvent<HTMLElement>) => {
+  const textEl = e.currentTarget.querySelector(".task-btn-copy-content");
+  const iconEl = e.currentTarget.querySelector("i.bi");
+  if (textEl) {
+    const htmlEl = textEl as HTMLElement;
+    const htmlIconEl = iconEl as HTMLElement;
+    const oldText = htmlEl.innerText;
+    const oldIconClz = htmlIconEl.className;
+    htmlEl.innerText = "Copied!";
+    htmlIconEl.className = `${ApplicationIcons.confirm}`;
+    setTimeout(() => {
+      window.getSelection()?.removeAllRanges();
+    }, 50);
+    setTimeout(() => {
+      htmlEl.innerText = oldText;
+      htmlIconEl.className = oldIconClz;
+    }, 1250);
+  }
+};
+
+const useResolvedTabs = ({
+  evalVersion,
+  evalStatus,
+  sampleMode,
+  samples,
+  selectedSample,
+  sampleStatus,
+  sampleError,
+  showingSampleDialog,
+  setShowingSampleDialog,
+  groupBy,
+  groupByOrder,
+  selectedSampleIndex,
+  setSelectedSampleIndex,
+  samplesDescriptor,
+  selectedSampleTab,
+  setSelectedSampleTab,
+  filter,
+  sort,
+  epoch,
+  sampleScrollPositionRef,
+  setSampleScrollPosition,
+  epochs,
+  setEpoch,
+  setFilter,
+  setSort,
+  score,
+  setScore,
+  scores,
+  evalSpec,
+  evalPlan,
+  evalResults,
+  evalStats,
+  evalError,
+  logFileName,
+  capabilities,
+  selectedTab,
+  refreshLog,
+}: WorkSpaceProps) => {
+  const sampleTabScrollRef = useRef<HTMLDivElement>(null);
+
+  const samplesTab =
+    sampleMode !== "none"
+      ? {
+          id: kEvalWorkspaceTabId,
+          scrollable: samples?.length === 1,
+          scrollRef: sampleTabScrollRef,
+          label: (samples || []).length > 1 ? "Samples" : "Sample",
+          content: () => (
+            <SamplesTab
+              sample={selectedSample}
+              sampleStatus={sampleStatus}
+              sampleError={sampleError}
+              showingSampleDialog={showingSampleDialog}
+              setShowingSampleDialog={setShowingSampleDialog}
+              samples={samples}
+              sampleMode={sampleMode}
+              groupBy={groupBy}
+              groupByOrder={groupByOrder}
+              selectedSampleIndex={selectedSampleIndex}
+              setSelectedSampleIndex={setSelectedSampleIndex}
+              sampleDescriptor={samplesDescriptor}
+              selectedSampleTab={selectedSampleTab}
+              setSelectedSampleTab={setSelectedSampleTab}
+              filter={filter}
+              epoch={epoch}
+              sampleScrollPositionRef={sampleScrollPositionRef}
+              setSampleScrollPosition={setSampleScrollPosition}
+              sampleTabScrollRef={sampleTabScrollRef}
+            />
+          ),
+          tools: () =>
+            sampleMode === "single" || !samplesDescriptor
+              ? undefined
+              : [
+                  <SampleTools
+                    key="sample-tools"
+                    epoch={epoch}
+                    epochs={epochs || 1}
+                    setEpoch={setEpoch}
+                    scoreFilter={filter}
+                    setScoreFilter={setFilter}
+                    sort={sort}
+                    setSort={setSort}
+                    score={score}
+                    setScore={setScore}
+                    scores={scores}
+                    sampleDescriptor={samplesDescriptor}
+                  />,
+                  evalStatus === "started" && (
+                    <ToolButton
+                      key="refresh"
+                      label="Refresh"
+                      icon={ApplicationIcons.refresh}
+                      onClick={refreshLog}
+                    />
+                  ),
+                ].filter(Boolean),
+        }
+      : null;
+
+  const configTab = {
+    id: kInfoWorkspaceTabId,
+    label: "Info",
+    scrollable: true,
+    content: () => (
+      <InfoTab
+        evalSpec={evalSpec}
+        evalPlan={evalPlan}
+        evalError={evalError}
+        evalResults={evalResults}
+        evalStats={evalStats}
+        samples={samples}
+      />
+    ),
+  };
+
+  const jsonTab = {
+    id: kJsonWorkspaceTabId,
+    label: "JSON",
+    scrollable: true,
+    content: () => {
+      const evalHeader = {
+        version: evalVersion,
+        status: evalStatus,
+        eval: evalSpec,
+        plan: evalPlan,
+        error: evalError,
+        results: evalResults,
+        stats: evalStats,
+      };
+      return (
+        <JsonTab
+          logFile={logFileName}
+          json={JSON.stringify(evalHeader, null, 2)}
+          capabilities={capabilities}
+          selected={selectedTab === kJsonWorkspaceTabId}
+        />
+      );
+    },
+    tools: () => [
+      <ToolButton
+        key="copy-json"
+        label="Copy JSON"
+        icon={ApplicationIcons.copy}
+        className={clsx("task-btn-json-copy", "clipboard-button")}
+        data-clipboard-target="#task-json-contents"
+        onClick={copyFeedback}
+      />,
+    ],
+  };
+
+  return useMemo(
+    () => ({
+      ...(samplesTab ? { samples: samplesTab } : {}),
+      config: configTab,
+      json: jsonTab,
+    }),
+    [samplesTab, configTab, jsonTab],
+  );
+};

inspect_ai/_view/www/src/workspace/WorkSpaceView.module.css

@@ -0,0 +1,33 @@
+.workspace {
+  padding-top: 0rem;
+  overflow-y: hidden;
+}
+
+.tabContainer {
+  padding: 0;
+  flex: 1;
+  display: flex;
+  flex-direction: column;
+  overflow-y: hidden;
+}
+
+.tabSet {
+  flex-wrap: nowrap;
+  padding: 0.5em 1em 0.5em 1em;
+  border-bottom: solid 1px var(--bs-border-color);
+  background: var(--bs-light);
+}
+
+.tabs {
+  padding: 0.3rem 0.3rem 0.3rem 0.3rem;
+  width: 5rem;
+  text-transform: uppercase;
+  border-radius: var(--bs-border-radius);
+  font-weight: 600;
+}
+
+.tabPanels {
+  flex: 1;
+  overflow-y: hidden;
+  display: flex;
+}