self-bench 0.3.5 → 0.3.6

package/src/{modal-executor.ts → sandbox/providers/modal/executor.ts}

@@ -1,12 +1,16 @@
+import { setTimeout as delay } from "node:timers/promises";
 import { type Image, ModalClient, type Secret } from "modal";
-import type { SelfBenchConfig } from "./config.js";
-import { InactivityTimeoutError, RollingOutput } from "./process.js";
-import type {
-  SandboxExecutor,
-  SandboxRequest,
-  SandboxResult,
-  SandboxRunOptions,
-} from "./sandbox.js";
+import type { SelfBenchConfig } from "../../../config.js";
+import { InactivityTimeoutError, RollingOutput } from "../../../process.js";
+import {
+  SandboxExecutionError,
+  type SandboxExecutor,
+  type SandboxRequest,
+  type SandboxResult,
+  type SandboxRunOptions,
+} from "../../contracts.js";
+
+const FAILURE_DRAIN_TIMEOUT_MS = 1_000;
 
 export class ModalSandboxExecutor implements SandboxExecutor {
   readonly #client: ModalClient;
@@ -22,10 +26,12 @@ export class ModalSandboxExecutor implements SandboxExecutor {
   }
 
   async run(request: SandboxRequest, options: SandboxRunOptions = {}): Promise<SandboxResult> {
+    options.signal?.throwIfAborted();
     const app = await this.#client.apps.fromName(this.#config.app, {
       createIfMissing: true,
       ...(this.#config.environment ? { environment: this.#config.environment } : {}),
     });
+    options.signal?.throwIfAborted();
     if (!this.#image) {
       this.#image = this.#buildImage();
     }
@@ -39,11 +45,14 @@ export class ModalSandboxExecutor implements SandboxExecutor {
       tags: { run_id: request.runId, stage: request.stage },
     });
 
+    let notifyFailure = (): void => {};
     const abort = () => {
+      notifyFailure();
       void sandbox.terminate();
     };
     options.signal?.addEventListener("abort", abort, { once: true });
     try {
+      options.signal?.throwIfAborted();
       for (const file of request.files ?? []) {
         if (typeof file.contents === "string") {
           await sandbox.filesystem.writeText(file.contents, file.path);
@@ -60,8 +69,12 @@ export class ModalSandboxExecutor implements SandboxExecutor {
       await process.closeStdin();
       const stdoutOutput = new RollingOutput();
       const stderrOutput = new RollingOutput();
+      const readers = new Set<ReadableStreamDefaultReader<string | Uint8Array>>();
       let inactivityTimer: ReturnType<typeof setTimeout> | undefined;
       let inactivityError: InactivityTimeoutError | undefined;
+      const failure = new Promise<void>((resolve) => {
+        notifyFailure = resolve;
+      });
       const clearInactivityTimer = (): void => {
         if (inactivityTimer) {
           clearTimeout(inactivityTimer);
@@ -78,7 +91,7 @@ export class ModalSandboxExecutor implements SandboxExecutor {
             `Modal sandbox ${sandbox.sandboxId} stage ${request.stage}`,
             request.inactivityTimeoutMs ?? 0,
           );
-          void sandbox.terminate();
+          notifyFailure();
         }, request.inactivityTimeoutMs);
         inactivityTimer.unref();
       };
@@ -88,6 +101,7 @@ export class ModalSandboxExecutor implements SandboxExecutor {
         output: RollingOutput,
       ): Promise<void> => {
         const reader = stream.getReader();
+        readers.add(reader);
         try {
           while (true) {
             const { value, done } = await reader.read();
@@ -102,25 +116,38 @@ export class ModalSandboxExecutor implements SandboxExecutor {
             }
           }
         } finally {
+          readers.delete(reader);
           reader.releaseLock();
         }
       };
+      let exitCode: number | undefined;
+      let processError: unknown;
+      let streamError: unknown;
+      const captureFailure = (error: unknown): void => {
+        streamError ??= error;
+        notifyFailure();
+      };
       armInactivityTimer();
-      const [exitCode, stdout, stderr] = await Promise.all([
-        process.wait(),
-        consume("stdout", process.stdout, stdoutOutput),
-        consume("stderr", process.stderr, stderrOutput),
-      ])
-        .then(([exitCode]) => {
-          if (inactivityError) {
-            throw inactivityError;
-          }
-          return [exitCode, stdoutOutput.text(), stderrOutput.text()] as const;
-        })
-        .catch((error: unknown) => {
-          throw inactivityError ?? error;
-        })
-        .finally(clearInactivityTimer);
+      const completion = Promise.all([
+        process.wait().then(
+          (value) => {
+            exitCode = value;
+          },
+          (error: unknown) => {
+            processError = error;
+            notifyFailure();
+          },
+        ),
+        consume("stdout", process.stdout, stdoutOutput).catch(captureFailure),
+        consume("stderr", process.stderr, stderrOutput).catch(captureFailure),
+      ]);
+      await Promise.race([completion, failure.then(() => delay(FAILURE_DRAIN_TIMEOUT_MS))]).finally(
+        clearInactivityTimer,
+      );
+      if (readers.size > 0) {
+        void Promise.allSettled([...readers].map((reader) => reader.cancel()));
+      }
+      const executionError = inactivityError ?? processError ?? streamError;
       const outputs: Record<string, Uint8Array> = {};
       for (const path of request.outputPaths ?? []) {
         try {
@@ -130,7 +157,21 @@ export class ModalSandboxExecutor implements SandboxExecutor {
           // absent output keeps the model or command failure diagnosable.
         }
       }
-      return { sandboxId: sandbox.sandboxId, exitCode, stdout, stderr, outputs };
+      const result = {
+        sandboxId: sandbox.sandboxId,
+        exitCode: exitCode ?? 1,
+        stdout: stdoutOutput.text(),
+        stderr: stderrOutput.text(),
+        outputs,
+      };
+      if (executionError) {
+        throw new SandboxExecutionError(
+          `${executionError instanceof Error ? executionError.message : String(executionError)}; sandbox ${sandbox.sandboxId}`,
+          result,
+          { cause: executionError },
+        );
+      }
+      return result;
     } finally {
       options.signal?.removeEventListener("abort", abort);
       await sandbox.terminate().catch(() => undefined);

package/src/{vercel-command.ts → sandbox/providers/vercel/command.ts}

@@ -1,7 +1,7 @@
 import type { Command, Session } from "@vercel/sandbox";
-import { InactivityTimeoutError, type RollingOutput } from "./process.js";
-import type { SandboxRequest, SandboxResult, SandboxRunOptions } from "./sandbox.js";
-import { VercelCommandStartError } from "./vercel-fetch.js";
+import { InactivityTimeoutError, type RollingOutput } from "../../../process.js";
+import type { SandboxRequest, SandboxResult, SandboxRunOptions } from "../../contracts.js";
+import { VercelCommandStartError } from "./fetch.js";
 
 export const VERCEL_WORK_DIRECTORY = "/work";
 

package/src/{vercel-executor.ts → sandbox/providers/vercel/executor.ts}

@@ -5,16 +5,16 @@ import {
   isDigestPinnedOciImage,
   type SelfBenchWorkerConfig,
   type VercelCredentials,
-} from "./config.js";
-import { RollingOutput } from "./process.js";
+} from "../../../config.js";
+import { RollingOutput } from "../../../process.js";
 import type {
   SandboxExecutor,
   SandboxRequest,
   SandboxResult,
   SandboxRunOptions,
-} from "./sandbox.js";
-import { executeVercelCommand, VERCEL_WORK_DIRECTORY } from "./vercel-command.js";
-import { preventAmbiguousVercelCommandStartRetries } from "./vercel-fetch.js";
+} from "../../contracts.js";
+import { executeVercelCommand, VERCEL_WORK_DIRECTORY } from "./command.js";
+import { preventAmbiguousVercelCommandStartRetries } from "./fetch.js";
 
 const CLEANUP_TIMEOUT_MS = 60_000;
 const CLEANUP_RETRY_DELAY_MS = 1_000;

package/src/{sandbox-timeout.ts → sandbox/timeout.ts}

@@ -3,7 +3,7 @@ import type {
   SandboxRequest,
   SandboxResult,
   SandboxRunOptions,
-} from "./sandbox.js";
+} from "./contracts.js";
 
 export const STANDARD_VERCEL_TIMEOUT_CAP_MS = 2 * 60 * 60 * 1_000;
 export const HOBBY_VERCEL_TIMEOUT_CAP_MS = 45 * 60 * 1_000;

package/src/{vercel-cli.ts → setup/vercel/cli.ts}

@@ -1,7 +1,7 @@
 import { spawn } from "node:child_process";
 import { resolve } from "node:path";
 import { z } from "zod";
-import { type CommandOutputHandler, type CommandResult, runCommand } from "./process.js";
+import { type CommandOutputHandler, type CommandResult, runCommand } from "../../process.js";
 
 const MINIMUM_CLI_VERSION = [59, 1, 3] as const;
 const PAGE_SIZE = "100";

package/src/setup/vercel/index.ts

@@ -0,0 +1,3 @@
+export { applyVercelProfile } from "./profile.js";
+export type { VercelSetupCli, VercelSetupServices } from "./setup.js";
+export { setupVercel } from "./setup.js";

package/src/{vercel-setup-probe.ts → setup/vercel/probe.ts}

@@ -1,7 +1,10 @@
 import { setTimeout as delay } from "node:timers/promises";
 import { APIError, Sandbox } from "@vercel/sandbox";
-import type { VercelCredentials } from "./config.js";
-import { HOBBY_VERCEL_TIMEOUT_CAP_MS, STANDARD_VERCEL_TIMEOUT_CAP_MS } from "./sandbox-timeout.js";
+import type { VercelCredentials } from "../../config.js";
+import {
+  HOBBY_VERCEL_TIMEOUT_CAP_MS,
+  STANDARD_VERCEL_TIMEOUT_CAP_MS,
+} from "../../sandbox/timeout.js";
 
 const PROBE_COMMAND_TIMEOUT_MS = 15_000;
 const CLEANUP_TIMEOUT_MS = 60_000;

package/src/{vercel-profile.ts → setup/vercel/profile.ts}

@@ -4,7 +4,10 @@ import { chmod, lstat, mkdir, readFile, rename, rm, writeFile } from "node:fs/pr
 import { homedir } from "node:os";
 import { join, resolve } from "node:path";
 import { z } from "zod";
-import { parseSandboxTimeoutCapText, STANDARD_VERCEL_TIMEOUT_CAP_MS } from "./sandbox-timeout.js";
+import {
+  parseSandboxTimeoutCapText,
+  STANDARD_VERCEL_TIMEOUT_CAP_MS,
+} from "../../sandbox/timeout.js";
 
 const CONFIG_FILE = "config.json";
 const CREDENTIALS_FILE = "credentials.json";

package/src/{vercel-runtime-image.ts → setup/vercel/runtime-image.ts}

@@ -1,10 +1,10 @@
 import { readFile } from "node:fs/promises";
 import { resolve } from "node:path";
-import { sha256 } from "./hash.js";
-import type { CommandOutputHandler } from "./process.js";
-import type { SetupPrompter } from "./terminal-prompts.js";
-import type { SetupReporter } from "./terminal-reporter.js";
-import type { VcrTag } from "./vercel-cli.js";
+import { sha256 } from "../../hash.js";
+import type { CommandOutputHandler } from "../../process.js";
+import type { SetupPrompter } from "../../terminal-prompts.js";
+import type { SetupReporter } from "../../terminal-reporter.js";
+import type { VcrTag } from "./cli.js";
 
 export const DEFAULT_VCR_REPOSITORY = "selfbench-runtime";
 

package/src/{vercel-setup.ts → setup/vercel/setup.ts}

@@ -1,15 +1,16 @@
 import { dirname, resolve } from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 import { fileURLToPath } from "node:url";
-import type { VercelCredentials } from "./config.js";
+import type { VercelCredentials } from "../../config.js";
 import {
   isInteractiveTerminal,
   SetupCanceledError,
   type SetupPrompter,
   TerminalPrompter,
-} from "./terminal-prompts.js";
-import { type SetupReporter, TerminalSetupReporter } from "./terminal-reporter.js";
-import { VercelCli, type VercelProject, type VercelTeam } from "./vercel-cli.js";
+} from "../../terminal-prompts.js";
+import { type SetupReporter, TerminalSetupReporter } from "../../terminal-reporter.js";
+import { VercelCli, type VercelProject, type VercelTeam } from "./cli.js";
+import { probeVercelCapability, type VercelCapability } from "./probe.js";
 import {
   loadVercelProfileData,
   saveVercelProfile,
@@ -17,14 +18,13 @@ import {
   type VercelProfile,
   type VercelProfileData,
   validateVercelProfileName,
-} from "./vercel-profile.js";
+} from "./profile.js";
 import {
   DEFAULT_VCR_REPOSITORY,
   ensureVercelRuntimeImage,
   type VercelRuntimeImageCli,
   vercelRuntimeFingerprint,
-} from "./vercel-runtime-image.js";
-import { probeVercelCapability, type VercelCapability } from "./vercel-setup-probe.js";
+} from "./runtime-image.js";
 
 const DEFAULT_PROFILE_NAME = "default";
 const DEFAULT_PROJECT_NAME = "selfbench-sandbox";
@@ -402,7 +402,7 @@ function imageDigest(image: string): string {
 }
 
 function defaultProjectRoot(): string {
-  return resolve(dirname(fileURLToPath(import.meta.url)), "..");
+  return resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
 }
 
 function errorMessage(error: unknown): string {