self-bench 0.3.3 → 0.3.5

package/src/terminal-prompts.ts

@@ -0,0 +1,197 @@
+import {
+  autocomplete,
+  cancel as cancelPrompt,
+  confirm as confirmPrompt,
+  isCancel,
+  password,
+  select as selectPrompt,
+  text as textPrompt,
+} from "@clack/prompts";
+
+export class SetupCanceledError extends Error {
+  constructor(message = "Setup canceled") {
+    super(message);
+    this.name = "SetupCanceledError";
+  }
+}
+
+export interface PromptChoice {
+  readonly value: string;
+  readonly label: string;
+  readonly hint?: string;
+}
+
+export interface SetupPrompter {
+  select(message: string, choices: readonly PromptChoice[], defaultValue?: string): Promise<string>;
+  search(message: string, choices: readonly PromptChoice[], defaultValue?: string): Promise<string>;
+  text(message: string, defaultValue?: string): Promise<string>;
+  secret(message: string): Promise<string>;
+  confirm(message: string, defaultValue: boolean): Promise<boolean>;
+}
+
+export class TerminalPrompter implements SetupPrompter {
+  readonly #input: NodeJS.ReadStream;
+  readonly #output: NodeJS.WriteStream;
+
+  constructor(
+    input: NodeJS.ReadStream = process.stdin,
+    output: NodeJS.WriteStream = process.stdout,
+  ) {
+    this.#input = input;
+    this.#output = output;
+  }
+
+  async select(
+    message: string,
+    choices: readonly PromptChoice[],
+    defaultValue?: string,
+  ): Promise<string> {
+    this.#assertChoices(message, choices);
+    return await this.#run((signal) =>
+      selectPrompt<string>({
+        message,
+        options: choices.map(toPromptOption),
+        maxItems: this.#visibleItemCount(),
+        showInstructions: true,
+        ...(defaultValue !== undefined && choices.some(({ value }) => value === defaultValue)
+          ? { initialValue: defaultValue }
+          : {}),
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async search(
+    message: string,
+    choices: readonly PromptChoice[],
+    defaultValue?: string,
+  ): Promise<string> {
+    this.#assertChoices(message, choices);
+    return await this.#run((signal) =>
+      autocomplete<string>({
+        message,
+        options: choices.map(toPromptOption),
+        placeholder: "Type to filter…",
+        maxItems: this.#visibleItemCount(),
+        filter: (query, option) =>
+          `${option.label ?? ""}\n${option.hint ?? ""}\n${option.value}`
+            .toLowerCase()
+            .includes(query.toLowerCase()),
+        ...(defaultValue !== undefined && choices.some(({ value }) => value === defaultValue)
+          ? { initialValue: defaultValue }
+          : {}),
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async text(message: string, defaultValue?: string): Promise<string> {
+    return await this.#run((signal) =>
+      textPrompt({
+        message,
+        ...(defaultValue === undefined ? {} : { placeholder: defaultValue, defaultValue }),
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async secret(message: string): Promise<string> {
+    if (!this.#input.isTTY || !this.#input.setRawMode) {
+      throw new Error("secure token input requires an interactive terminal");
+    }
+    return await this.#run((signal) =>
+      password({
+        message,
+        // An empty mask keeps both the token and its length out of terminal output.
+        mask: "",
+        validate: (value) => (value?.trim() ? undefined : "Enter a project-scoped token."),
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async confirm(message: string, defaultValue: boolean): Promise<boolean> {
+    return await this.#run((signal) =>
+      confirmPrompt({
+        message,
+        initialValue: defaultValue,
+        input: this.#input,
+        output: this.#output,
+        signal,
+      }),
+    );
+  }
+
+  async #run<T>(prompt: (signal: AbortSignal) => Promise<T | symbol>): Promise<T> {
+    const controller = new AbortController();
+    const wasRaw = this.#input.isRaw;
+    const wasFlowing = this.#input.readableFlowing;
+    let inputFailure: Error | undefined;
+    const abort = (error: Error): void => {
+      inputFailure = error;
+      controller.abort(error);
+    };
+    const onEnd = (): void => abort(new SetupCanceledError("Setup canceled at end of input"));
+    const onError = (error: Error): void => abort(error);
+    this.#input.once("end", onEnd);
+    this.#input.once("error", onError);
+    if (this.#input.readableEnded) {
+      onEnd();
+    }
+
+    try {
+      const result = await prompt(controller.signal);
+      if (isCancel(result)) {
+        const error = inputFailure ?? new SetupCanceledError();
+        if (error instanceof SetupCanceledError) {
+          cancelPrompt("Setup canceled.", { output: this.#output, withGuide: true });
+        }
+        throw error;
+      }
+      return result;
+    } finally {
+      this.#input.off("end", onEnd);
+      this.#input.off("error", onError);
+      if (this.#input.isTTY && this.#input.setRawMode && this.#input.isRaw !== wasRaw) {
+        this.#input.setRawMode(Boolean(wasRaw));
+      }
+      if (wasFlowing !== true) {
+        this.#input.pause();
+      }
+    }
+  }
+
+  #assertChoices(message: string, choices: readonly PromptChoice[]): void {
+    if (choices.length === 0) {
+      throw new Error(`${message} has no available choices`);
+    }
+  }
+
+  #visibleItemCount(): number {
+    return Math.max(3, Math.min(8, (this.#output.rows ?? 20) - 6));
+  }
+}
+
+export function isInteractiveTerminal(): boolean {
+  return Boolean(process.stdin.isTTY && process.stdout.isTTY);
+}
+
+function toPromptOption(choice: PromptChoice): {
+  readonly value: string;
+  readonly label: string;
+  readonly hint?: string;
+} {
+  return {
+    value: choice.value,
+    label: choice.label,
+    ...(choice.hint ? { hint: choice.hint } : {}),
+  };
+}

package/src/terminal-reporter.ts

@@ -0,0 +1,126 @@
+import type { Writable } from "node:stream";
+import { stripVTControlCharacters } from "node:util";
+import { cancel as cancelPrompt, intro, log, outro, spinner } from "@clack/prompts";
+import { type CommandOutputHandler, RollingOutput } from "./process.js";
+import { SetupCanceledError } from "./terminal-prompts.js";
+
+export interface SetupTaskLabels {
+  readonly pending: string;
+  readonly success: string;
+  readonly failure: string;
+}
+
+export interface SetupReporter {
+  intro(title: string): void;
+  message(message: string): void;
+  warn(message: string): void;
+  cancel(message: string): void;
+  task<T>(
+    labels: SetupTaskLabels,
+    operation: (onOutput: CommandOutputHandler) => Promise<T>,
+  ): Promise<T>;
+  finish(title: string, details: readonly string[]): void;
+}
+
+export class TerminalSetupReporter implements SetupReporter {
+  readonly #output: Writable;
+  readonly #errorOutput: Writable;
+  readonly #verbose: boolean;
+
+  constructor(
+    options: {
+      readonly output?: Writable;
+      readonly errorOutput?: Writable;
+      readonly verbose?: boolean;
+    } = {},
+  ) {
+    this.#output = options.output ?? process.stdout;
+    this.#errorOutput = options.errorOutput ?? process.stderr;
+    this.#verbose = options.verbose ?? false;
+  }
+
+  intro(title: string): void {
+    intro(title, { output: this.#output, withGuide: true });
+  }
+
+  message(message: string): void {
+    log.message(message, { output: this.#output, withGuide: true });
+  }
+
+  warn(message: string): void {
+    log.warn(message, { output: this.#output, withGuide: true });
+  }
+
+  cancel(message: string): void {
+    cancelPrompt(message, { output: this.#output, withGuide: true });
+  }
+
+  async task<T>(
+    labels: SetupTaskLabels,
+    operation: (onOutput: CommandOutputHandler) => Promise<T>,
+  ): Promise<T> {
+    const captured = new RollingOutput();
+    const onOutput: CommandOutputHandler = (stream, chunk) => {
+      captured.push(Buffer.from(chunk));
+      if (this.#verbose) {
+        (stream === "stderr" ? this.#errorOutput : this.#output).write(chunk);
+      }
+    };
+
+    if (this.#verbose) {
+      log.step(labels.pending, { output: this.#output, withGuide: true });
+      try {
+        const result = await operation(onOutput);
+        log.success(labels.success, { output: this.#output, withGuide: true });
+        return result;
+      } catch (error) {
+        if (error instanceof SetupCanceledError) {
+          this.cancel("Setup canceled.");
+          throw error;
+        }
+        log.error(labels.failure, { output: this.#output, withGuide: true });
+        throw error;
+      }
+    }
+
+    const progress = spinner({ indicator: "timer", output: this.#output, withGuide: true });
+    progress.start(labels.pending);
+    try {
+      const result = await operation(onOutput);
+      progress.stop(labels.success);
+      return result;
+    } catch (error) {
+      if (error instanceof SetupCanceledError) {
+        progress.cancel("Setup canceled.");
+        throw error;
+      }
+      progress.error(labels.failure);
+      this.#showFailureOutput(captured.text());
+      throw error;
+    }
+  }
+
+  finish(title: string, details: readonly string[]): void {
+    log.success(title, { output: this.#output, withGuide: true });
+    log.message([...details], {
+      output: this.#output,
+      spacing: 0,
+      withGuide: true,
+    });
+    outro("Vercel Sandbox is ready.", { output: this.#output, withGuide: true });
+  }
+
+  #showFailureOutput(value: string): void {
+    const cleaned = stripVTControlCharacters(value)
+      .replace(/\r(?!\n)/g, "\n")
+      .trim();
+    if (!cleaned) {
+      return;
+    }
+    log.message(["Command output:", ...cleaned.split("\n")], {
+      output: this.#output,
+      spacing: 0,
+      withGuide: true,
+    });
+  }
+}

package/src/validate-main.ts

@@ -3,9 +3,11 @@
 import { access, mkdir, readdir, writeFile } from "node:fs/promises";
 import { basename, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
+import { harborChildEnvironment } from "./harbor-environment.js";
 import { harborInfrastructureError, readHarborJobResult } from "./harbor-results.js";
 import { parallelMap } from "./parallel.js";
 import { runCommand } from "./process.js";
+import { type HarborEnvironment, isHarborEnvironment } from "./providers.js";
 
 const parsed = parseArgs({
   options: {
@@ -33,7 +35,7 @@ Options:
 const tasksDirectory = resolve(parsed.values.tasks ?? fail("--tasks is required"));
 const jobsDirectory = resolve(parsed.values.jobs ?? fail("--jobs is required"));
 const environment = parsed.values.environment ?? "modal";
-if (environment !== "modal" && environment !== "docker") {
+if (!isHarborEnvironment(environment)) {
   throw new Error("--environment must be modal or docker");
 }
 const concurrency = positiveInteger(parsed.values.concurrency, "--concurrency");
@@ -90,7 +92,7 @@ async function runGate(
   taskDirectory: string,
   taskId: string,
   agent: "nop" | "oracle",
-  environment: "docker" | "modal",
+  environment: HarborEnvironment,
   jobsDirectory: string,
 ): Promise<{ readonly passed: boolean; readonly rewards: Readonly<Record<string, number>> }> {
   const jobName = `${taskId}-${agent}-${crypto.randomUUID().slice(0, 8)}`;
@@ -116,7 +118,11 @@ async function runGate(
       "--yes",
       "--quiet",
     ],
-    { allowFailure: true, timeoutMs: 4 * 60 * 60 * 1000 },
+    {
+      allowFailure: true,
+      env: harborChildEnvironment(),
+      timeoutMs: 4 * 60 * 60 * 1000,
+    },
   );
   if (process.exitCode !== 0) {
     throw new Error(`Harbor ${agent} exited ${process.exitCode}: ${process.stderr.slice(-1_000)}`);