inspect-ai 0.3.81__py3-none-any.whl → 0.3.83__py3-none-any.whl

inspect_ai/_view/www/src/utils/logger.ts

@@ -0,0 +1,52 @@
+// This will be replaced at build time with a boolean value
+declare const __DEV_WATCH__: boolean;
+declare const __LOGGING_FILTER__: string;
+
+const getEnabledNamespaces = () => {
+  // Split by comma and filter out empty strings
+  return __LOGGING_FILTER__
+    .split(",")
+    .map((ns) => ns.trim())
+    .filter(Boolean);
+};
+
+const ENABLED_NAMESPACES = new Set<string>(getEnabledNamespaces());
+const filterNameSpace = (namespace: string) => {
+  if (ENABLED_NAMESPACES.has("*")) return true;
+
+  return ENABLED_NAMESPACES.has(namespace);
+};
+
+// Create a logger for a specific namespace
+export const createLogger = (namespace: string) => {
+  // Logger functions that only activate in dev-watch mode
+  const logger = {
+    debug: (message: string, ...args: any[]) => {
+      if (__DEV_WATCH__ && filterNameSpace(namespace))
+        console.debug(`[${namespace}] ${message}`, ...args);
+    },
+
+    info: (message: string, ...args: any[]) => {
+      if (__DEV_WATCH__ && filterNameSpace(namespace))
+        console.info(`[${namespace}] ${message}`, ...args);
+    },
+
+    warn: (message: string, ...args: any[]) => {
+      if (__DEV_WATCH__ && filterNameSpace(namespace))
+        console.warn(`[${namespace}] ${message}`, ...args);
+    },
+
+    // Always log errors, even in production
+    error: (message: string, ...args: any[]) => {
+      console.error(`[${namespace}] ${message}`, ...args);
+    },
+
+    // Lazy evaluation for expensive logs
+    debugIf: (fn: () => string) => {
+      if (__DEV_WATCH__ && filterNameSpace(namespace))
+        console.debug(`[${namespace}] ${fn()}`);
+    },
+  };
+
+  return logger;
+};

inspect_ai/_view/www/src/utils/polling.ts

@@ -0,0 +1,100 @@
+import { Timeout } from "../types/log";
+import { createLogger } from "./logger";
+
+export interface PollingOptions {
+  maxRetries: number;
+  interval: number;
+}
+
+export interface Polling {
+  name: string;
+  start: () => void;
+  stop: () => void;
+}
+
+export const createPolling = (
+  name: string,
+  callback: () => Promise<boolean>,
+  options: PollingOptions,
+): Polling => {
+  const log = createLogger(`Polling ${name}`);
+
+  const { maxRetries, interval } = options;
+  let timeoutId: Timeout = null;
+  let retryCount = 0;
+  // Are we currently polling
+  let isPolling = false;
+
+  // Are we stopped (e.g. cleaning up)
+  let isStopped = false;
+
+  const calculateBackoff = (retryCount: number) => {
+    return Math.min(interval * Math.pow(2, retryCount) * 1000, 60000);
+  };
+
+  const stop = () => {
+    if (timeoutId) {
+      clearTimeout(timeoutId);
+      timeoutId = null;
+    }
+    log.debug("Stop Polling");
+    isPolling = false;
+    isStopped = true;
+  };
+
+  const poll = async () => {
+    try {
+      log.debug("Poll");
+      // Don't proceed if polling has been stopped
+      if (!isPolling || isStopped) {
+        return;
+      }
+
+      const shouldContinue = await callback();
+      if (shouldContinue === false) {
+        stop();
+        return;
+      }
+
+      // Reset retry count on success
+      retryCount = 0;
+      if (!isPolling || isStopped) {
+        return;
+      }
+      timeoutId = setTimeout(poll, interval * 1000);
+    } catch (e) {
+      // Don't retry if polling has been stopped
+      if (!isPolling || isStopped) {
+        return;
+      }
+      log.debug("Polling error occurred", e);
+
+      retryCount += 1;
+
+      if (retryCount >= maxRetries) {
+        stop();
+        throw new Error(
+          `Gave up polling ${name} after ${maxRetries} attempts.`,
+        );
+      }
+
+      const backoffTime = calculateBackoff(retryCount);
+      log.debug(
+        `Retry ${retryCount}/${maxRetries}, backoff: ${backoffTime / 1000}s`,
+      );
+      timeoutId = setTimeout(poll, backoffTime);
+    }
+  };
+
+  const start = () => {
+    if (isPolling) {
+      return;
+    }
+    log.debug("Start Polling");
+    isPolling = true;
+    isStopped = false;
+    poll();
+  };
+
+  return { name, start, stop };
+};

inspect_ai/_view/www/src/utils/react.ts

@@ -0,0 +1,30 @@
+import { useEffect, useRef } from "react";
+
+export const useWhyDidYouUpdate = (componentName: string, props: any) => {
+  const previousProps = useRef(props);
+
+  useEffect(() => {
+    if (previousProps.current !== props) {
+      const changedProps = Object.entries(props).reduce(
+        (diff, [key, value]) => {
+          if (previousProps.current[key] !== value) {
+            diff[key] = {
+              before: previousProps.current[key],
+              after: value,
+            };
+          }
+          return diff;
+        },
+        {} as Record<string, unknown>,
+      );
+
+      if (Object.keys(changedProps).length > 0) {
+        console.log(`[${componentName}] props changed:`, changedProps);
+      } else {
+        console.log(`[${componentName}] no props changed`);
+      }
+    }
+
+    previousProps.current = props;
+  }, [props, componentName]);
+};

inspect_ai/_view/www/src/utils/vscode.ts

@@ -2,7 +2,7 @@
  * Type definition for the VS Code API object
  * Note: This is a minimal definition - expand based on your needs
  */
-interface VSCodeApi {
+export interface VSCodeApi {
   postMessage(message: unknown): void;
   getState(): unknown;
   setState(state: unknown): void;

inspect_ai/_view/www/src/workspace/WorkSpace.tsx

@@ -1,31 +1,28 @@
 import { ApplicationIcons } from "../appearance/icons";
 import { ToolButton } from "../components/ToolButton";
-import { SampleTools } from "../samples/SamplesTools";
+import { SampleTools, ScoreFilterTools } from "../samples/SamplesTools";
 import { JsonTab } from "./tabs/JsonTab";
 import { SamplesTab } from "./tabs/SamplesTab";
 
 import clsx from "clsx";
-import { FC, MouseEvent, RefObject, useEffect, useMemo, useRef } from "react";
-import { SampleSummary } from "../api/types.ts";
+import { FC, MouseEvent, useEffect, useMemo, useRef } from "react";
+import { RunningMetric } from "../api/types.ts";
 import {
   kEvalWorkspaceTabId,
   kInfoWorkspaceTabId,
   kJsonWorkspaceTabId,
 } from "../constants";
-import { SamplesDescriptor } from "../samples/descriptor/samplesDescriptor";
 import {
-  Capabilities,
-  CurrentLog,
-  SampleMode,
-  ScoreFilter,
-  ScoreLabel,
-} from "../types.ts";
+  useFilteredSamples,
+  useSampleDescriptor,
+  useTotalSampleCount,
+} from "../state/hooks.ts";
+import { useStore } from "../state/store.ts";
+import { CurrentLog } from "../types.ts";
 import {
-  Epochs,
   EvalError,
   EvalPlan,
   EvalResults,
-  EvalSample,
   EvalSpec,
   EvalStats,
   Status,
@@ -35,7 +32,6 @@ import { WorkSpaceView } from "./WorkSpaceView.tsx";
 
 interface WorkSpaceProps {
   task_id?: string;
-  logFileName?: string;
   evalError?: EvalError;
   evalStatus?: Status;
   evalVersion?: number;
@@ -43,42 +39,10 @@ interface WorkSpaceProps {
   evalPlan?: EvalPlan;
   evalStats?: EvalStats;
   evalResults?: EvalResults;
+  runningMetrics?: RunningMetric[];
   log?: CurrentLog;
-  samples?: SampleSummary[];
-  sampleMode: SampleMode;
-  groupBy: "none" | "epoch" | "sample";
-  groupByOrder: "asc" | "desc";
-  selectedSample?: EvalSample;
-  sampleStatus: string;
-  sampleError?: Error;
   showToggle: boolean;
-  refreshLog: () => Promise<void>;
-  capabilities: Capabilities;
-  selectedSampleIndex: number;
-  samplesDescriptor?: SamplesDescriptor;
-  setSelectedSampleIndex: (index: number) => void;
-  selectedSampleTab?: string;
-  setSelectedSampleTab: (tab: string) => void;
-  sort: string;
-  setSort: (sort: string) => void;
-  epochs?: Epochs;
-  epoch: string;
-  showingSampleDialog: boolean;
-  setShowingSampleDialog: (showing: boolean) => void;
-  setEpoch: (epoch: string) => void;
-  filter: ScoreFilter;
-  setFilter: (filter: ScoreFilter) => void;
-  score?: ScoreLabel;
-  setScore: (score: ScoreLabel) => void;
-  scores: ScoreLabel[];
-  offcanvas: boolean;
-  setOffcanvas: (offcanvas: boolean) => void;
-  selectedTab: string;
-  setSelectedTab: (id: string) => void;
-  sampleScrollPositionRef: RefObject<number>;
-  setSampleScrollPosition: (position: number) => void;
-  workspaceTabScrollPositionRef: RefObject<Record<string, number>>;
-  setWorkspaceTabScrollPosition: (tab: string, position: number) => void;
+  refreshLog: () => void;
 }
 
 /**
@@ -88,20 +52,12 @@ export const WorkSpace: FC<WorkSpaceProps> = (props) => {
   const {
     task_id,
     evalStatus,
-    logFileName,
     evalSpec,
     evalPlan,
     evalStats,
     evalResults,
-    samples,
+    runningMetrics,
     showToggle,
-    offcanvas,
-    setOffcanvas,
-    samplesDescriptor,
-    selectedTab,
-    setSelectedTab,
-    workspaceTabScrollPositionRef,
-    setWorkspaceTabScrollPosition,
   } = props;
 
   const divRef = useRef<HTMLDivElement>(null);
@@ -121,27 +77,20 @@ export const WorkSpace: FC<WorkSpaceProps> = (props) => {
 
   return (
     <WorkSpaceView
-      logFileName={logFileName}
       divRef={divRef}
       evalSpec={evalSpec}
       evalPlan={evalPlan}
       evalResults={evalResults}
+      runningMetrics={runningMetrics}
       evalStats={evalStats}
-      samples={samples}
-      evalDescriptor={samplesDescriptor?.evalDescriptor}
       status={evalStatus}
       tabs={resolvedTabs}
-      selectedTab={selectedTab}
       showToggle={showToggle}
-      offcanvas={offcanvas}
-      setSelectedTab={setSelectedTab}
-      workspaceTabScrollPositionRef={workspaceTabScrollPositionRef}
-      setWorkspaceTabScrollPosition={setWorkspaceTabScrollPosition}
-      setOffcanvas={setOffcanvas}
     />
   );
 };
 
+// Helper function for copy feedback
 const copyFeedback = (e: MouseEvent<HTMLElement>) => {
   const textEl = e.currentTarget.querySelector(".task-btn-copy-content");
   const iconEl = e.currentTarget.querySelector("i.bi");
@@ -162,164 +111,182 @@ const copyFeedback = (e: MouseEvent<HTMLElement>) => {
   }
 };
 
-const useResolvedTabs = ({
-  evalVersion,
-  evalStatus,
-  sampleMode,
-  samples,
-  selectedSample,
-  sampleStatus,
-  sampleError,
-  showingSampleDialog,
-  setShowingSampleDialog,
-  groupBy,
-  groupByOrder,
-  selectedSampleIndex,
-  setSelectedSampleIndex,
-  samplesDescriptor,
-  selectedSampleTab,
-  setSelectedSampleTab,
-  filter,
-  sort,
-  epoch,
-  sampleScrollPositionRef,
-  setSampleScrollPosition,
-  epochs,
-  setEpoch,
-  setFilter,
-  setSort,
-  score,
-  setScore,
-  scores,
-  evalSpec,
-  evalPlan,
-  evalResults,
-  evalStats,
-  evalError,
-  logFileName,
-  capabilities,
-  selectedTab,
-  refreshLog,
-}: WorkSpaceProps) => {
-  const sampleTabScrollRef = useRef<HTMLDivElement>(null);
+// Individual hook for Samples tab
+export const useSamplesTabConfig = (
+  evalStatus: Status | undefined,
+  refreshLog: () => void,
+) => {
+  const totalSampleCount = useTotalSampleCount();
+  const samplesDescriptor = useSampleDescriptor();
+  const sampleSummaries = useFilteredSamples();
+  const streamSamples = useStore((state) => state.capabilities.streamSamples);
+
+  return useMemo(() => {
+    return {
+      id: kEvalWorkspaceTabId,
+      scrollable: false,
+      label: totalSampleCount > 1 ? "Samples" : "Sample",
+      component: SamplesTab,
+      componentProps: {
+        running: evalStatus === "started",
+      },
+      tools: () =>
+        !samplesDescriptor
+          ? undefined
+          : totalSampleCount === 1
+            ? [<ScoreFilterTools />]
+            : [
+                <SampleTools
+                  samples={sampleSummaries || []}
+                  key="sample-tools"
+                />,
+                evalStatus === "started" && !streamSamples && (
+                  <ToolButton
+                    key="refresh"
+                    label="Refresh"
+                    icon={ApplicationIcons.refresh}
+                    onClick={refreshLog}
+                  />
+                ),
+              ],
+    };
+  }, [
+    evalStatus,
+    refreshLog,
+    sampleSummaries,
+    samplesDescriptor,
+    totalSampleCount,
+  ]);
+};
+
+// Individual hook for Info tab
+export const useInfoTabConfig = (
+  evalSpec: EvalSpec | undefined,
+  evalPlan: EvalPlan | undefined,
+  evalError: EvalError | undefined,
+  evalResults: EvalResults | undefined,
+  evalStats: EvalStats | undefined,
+) => {
+  const totalSampleCount = useTotalSampleCount();
+  return useMemo(() => {
+    return {
+      id: kInfoWorkspaceTabId,
+      label: "Info",
+      scrollable: true,
+      component: InfoTab,
+      componentProps: {
+        evalSpec,
+        evalPlan,
+        evalError,
+        evalResults,
+        evalStats,
+        sampleCount: totalSampleCount,
+      },
+    };
+  }, [evalSpec, evalPlan, evalError, evalResults, evalStats, totalSampleCount]);
+};
+
+// Individual hook for JSON tab
+export const useJsonTabConfig = (
+  evalVersion: number | undefined,
+  evalStatus: Status | undefined,
+  evalSpec: EvalSpec | undefined,
+  evalPlan: EvalPlan | undefined,
+  evalError: EvalError | undefined,
+  evalResults: EvalResults | undefined,
+  evalStats: EvalStats | undefined,
+) => {
+  const selectedLogFile = useStore((state) =>
+    state.logsActions.getSelectedLogFile(),
+  );
+  const selectedTab = useStore((state) => state.app.tabs.workspace);
+
+  return useMemo(() => {
+    const evalHeader = {
+      version: evalVersion,
+      status: evalStatus,
+      eval: evalSpec,
+      plan: evalPlan,
+      error: evalError,
+      results: evalResults,
+      stats: evalStats,
+    };
+
+    return {
+      id: kJsonWorkspaceTabId,
+      label: "JSON",
+      scrollable: true,
+      component: JsonTab,
+      componentProps: {
+        logFile: selectedLogFile,
+        json: JSON.stringify(evalHeader, null, 2),
+        selected: selectedTab === kJsonWorkspaceTabId,
+      },
+      tools: () => [
+        <ToolButton
+          key="copy-json"
+          label="Copy JSON"
+          icon={ApplicationIcons.copy}
+          className={clsx("task-btn-json-copy", "clipboard-button")}
+          data-clipboard-target="#task-json-contents"
+          onClick={copyFeedback}
+        />,
+      ],
+    };
+  }, [
+    selectedLogFile,
+    evalVersion,
+    evalStatus,
+    evalSpec,
+    evalPlan,
+    evalError,
+    evalResults,
+    evalStats,
+    selectedTab,
+  ]);
+};
+
+// Main hook combining all tab configs
+export const useResolvedTabs = (props: WorkSpaceProps) => {
+  const {
+    evalVersion,
+    evalStatus,
+    evalSpec,
+    evalPlan,
+    evalResults,
+    evalStats,
+    evalError,
+    refreshLog,
+  } = props;
 
-  const samplesTab =
-    sampleMode !== "none"
-      ? {
-          id: kEvalWorkspaceTabId,
-          scrollable: samples?.length === 1,
-          scrollRef: sampleTabScrollRef,
-          label: (samples || []).length > 1 ? "Samples" : "Sample",
-          content: () => (
-            <SamplesTab
-              sample={selectedSample}
-              sampleStatus={sampleStatus}
-              sampleError={sampleError}
-              showingSampleDialog={showingSampleDialog}
-              setShowingSampleDialog={setShowingSampleDialog}
-              samples={samples}
-              sampleMode={sampleMode}
-              groupBy={groupBy}
-              groupByOrder={groupByOrder}
-              selectedSampleIndex={selectedSampleIndex}
-              setSelectedSampleIndex={setSelectedSampleIndex}
-              sampleDescriptor={samplesDescriptor}
-              selectedSampleTab={selectedSampleTab}
-              setSelectedSampleTab={setSelectedSampleTab}
-              filter={filter}
-              epoch={epoch}
-              sampleScrollPositionRef={sampleScrollPositionRef}
-              setSampleScrollPosition={setSampleScrollPosition}
-              sampleTabScrollRef={sampleTabScrollRef}
-            />
-          ),
-          tools: () =>
-            sampleMode === "single" || !samplesDescriptor
-              ? undefined
-              : [
-                  <SampleTools
-                    key="sample-tools"
-                    epoch={epoch}
-                    epochs={epochs || 1}
-                    setEpoch={setEpoch}
-                    scoreFilter={filter}
-                    setScoreFilter={setFilter}
-                    sort={sort}
-                    setSort={setSort}
-                    score={score}
-                    setScore={setScore}
-                    scores={scores}
-                    sampleDescriptor={samplesDescriptor}
-                  />,
-                  evalStatus === "started" && (
-                    <ToolButton
-                      key="refresh"
-                      label="Refresh"
-                      icon={ApplicationIcons.refresh}
-                      onClick={refreshLog}
-                    />
-                  ),
-                ].filter(Boolean),
-        }
-      : null;
+  // Use individual tab config hooks
+  const samplesTabConfig = useSamplesTabConfig(evalStatus, refreshLog);
 
-  const configTab = {
-    id: kInfoWorkspaceTabId,
-    label: "Info",
-    scrollable: true,
-    content: () => (
-      <InfoTab
-        evalSpec={evalSpec}
-        evalPlan={evalPlan}
-        evalError={evalError}
-        evalResults={evalResults}
-        evalStats={evalStats}
-        samples={samples}
-      />
-    ),
-  };
+  const configTabConfig = useInfoTabConfig(
+    evalSpec,
+    evalPlan,
+    evalError,
+    evalResults,
+    evalStats,
+  );
 
-  const jsonTab = {
-    id: kJsonWorkspaceTabId,
-    label: "JSON",
-    scrollable: true,
-    content: () => {
-      const evalHeader = {
-        version: evalVersion,
-        status: evalStatus,
-        eval: evalSpec,
-        plan: evalPlan,
-        error: evalError,
-        results: evalResults,
-        stats: evalStats,
-      };
-      return (
-        <JsonTab
-          logFile={logFileName}
-          json={JSON.stringify(evalHeader, null, 2)}
-          capabilities={capabilities}
-          selected={selectedTab === kJsonWorkspaceTabId}
-        />
-      );
-    },
-    tools: () => [
-      <ToolButton
-        key="copy-json"
-        label="Copy JSON"
-        icon={ApplicationIcons.copy}
-        className={clsx("task-btn-json-copy", "clipboard-button")}
-        data-clipboard-target="#task-json-contents"
-        onClick={copyFeedback}
-      />,
-    ],
-  };
+  const jsonTabConfig = useJsonTabConfig(
+    evalVersion,
+    evalStatus,
+    evalSpec,
+    evalPlan,
+    evalError,
+    evalResults,
+    evalStats,
+  );
 
+  // Combine all tab configs
   return useMemo(
     () => ({
-      ...(samplesTab ? { samples: samplesTab } : {}),
-      config: configTab,
-      json: jsonTab,
+      ...(samplesTabConfig ? { samples: samplesTabConfig } : {}),
+      config: configTabConfig,
+      json: jsonTabConfig,
     }),
-    [samplesTab, configTab, jsonTab],
+    [samplesTabConfig, configTabConfig, jsonTabConfig],
   );
 };