pi-harness-runtime 1.1.385 → 1.1.387

package/packages/jev-judge/src/provider-error.ts

@@ -0,0 +1,190 @@
+/**
+ * Provider errors for the Jev judge.
+ *
+ * The judge can talk to more than one provider (TypeSafe direct API first,
+ * OpenRouter as a fallback). A failure therefore carries the information the
+ * fallback logic needs: which provider failed, whether the failure was an
+ * HTTP error, and whether ANOTHER provider could still serve the request.
+ */
+
+import type { JevProviderName } from "./types.js";
+
+export interface JevProviderErrorOptions {
+	provider: JevProviderName;
+	/** HTTP status, when the failure came from an HTTP response. */
+	status?: number;
+	/**
+	 * True when another provider could still serve the request.
+	 *
+	 * Retryable: network failures, timeouts, rate limits, auth failures and
+	 * 5xx. NOT retryable: 400/422 - those are contract errors in the request
+	 * we build, and falling back would hide the bug.
+	 */
+	retryable: boolean;
+	cause?: unknown;
+}
+
+/**
+ * Error raised by a single Jev provider call.
+ */
+export class JevProviderError extends Error {
+	readonly provider: JevProviderName;
+	readonly status?: number;
+	readonly retryable: boolean;
+
+	constructor(message: string, options: JevProviderErrorOptions) {
+		super(message, options.cause !== undefined ? { cause: options.cause } : undefined);
+		this.name = "JevProviderError";
+		this.provider = options.provider;
+		this.status = options.status;
+		this.retryable = options.retryable;
+	}
+}
+
+/**
+ * HTTP statuses where another provider (or a later attempt) can still help.
+ */
+export function isRetryableStatus(status: number): boolean {
+	if (status === 401 || status === 402 || status === 403) return true; // auth / quota
+	if (status === 408 || status === 429) return true; // timeout / rate limit
+	if (status >= 500) return true; // server side
+	return false;
+}
+
+/**
+ * Extract an HTTP status from an unknown error value.
+ */
+export function extractStatus(error: unknown): number | undefined {
+	if (typeof error !== "object" || error === null) return undefined;
+	const status = (error as { status?: unknown }).status;
+	return typeof status === "number" ? status : undefined;
+}
+
+/**
+ * Error names of the known TRANSPORT failures: the request never reached a
+ * server that could reject it, so another provider can still serve it.
+ */
+const TRANSPORT_ERROR_NAMES = new Set([
+	"AbortError",
+	"TimeoutError",
+	"APIConnectionError",
+	"APIConnectionTimeoutError",
+	"APITimeoutError",
+	"FetchError",
+	"ECONNABORTED",
+	"ECONNREFUSED",
+	"ECONNRESET",
+	"EAI_AGAIN",
+	"ENOTFOUND",
+	"ETIMEDOUT",
+	"UND_ERR_CONNECT_TIMEOUT",
+	"UND_ERR_HEADERS_TIMEOUT",
+	"UND_ERR_SOCKET",
+]);
+
+/** Error codes of the known transport failures. */
+const TRANSPORT_ERROR_CODES = new Set([
+	"ECONNABORTED",
+	"ECONNREFUSED",
+	"ECONNRESET",
+	"EAI_AGAIN",
+	"ENETDOWN",
+	"ENETUNREACH",
+	"EHOSTDOWN",
+	"EHOSTUNREACH",
+	"ENOTFOUND",
+	"EPROTO",
+	"EPIPE",
+	"ETIMEDOUT",
+	"ERR_STREAM_PREMATURE_CLOSE",
+	"UND_ERR_CONNECT_TIMEOUT",
+	"UND_ERR_HEADERS_TIMEOUT",
+	"UND_ERR_SOCKET",
+	// Certificate verification failures: the endpoint is unreachable for us.
+	"CERT_HAS_EXPIRED",
+	"SELF_SIGNED_CERT_IN_CHAIN",
+	"DEPTH_ZERO_SELF_SIGNED_CERT",
+	"UNABLE_TO_VERIFY_LEAF_SIGNATURE",
+]);
+
+/**
+ * Prefixes of the error codes that always describe a transport failure.
+ *
+ * undici reports its own failures as UND_ERR_*, the TLS stack as ERR_TLS_*
+ * and the network stack as ECONN / EHOST / ENET.
+ */
+const TRANSPORT_ERROR_CODE_PREFIXES = [
+	"UND_ERR_",
+	"ERR_TLS_",
+	"ECONN",
+	"EHOST",
+	"ENET",
+	"CERT_",
+	"SELF_SIGNED_CERT_",
+	"DEPTH_ZERO_",
+	"UNABLE_TO_VERIFY_",
+];
+
+/**
+ * True when the error is a known transport failure (network, timeout, abort).
+ *
+ * Only these are retryable without an HTTP status: they never reached a
+ * server that could have rejected the request. Everything else without a
+ * status (a bug in our own code, a malformed response, a parse error) is NOT
+ * retryable - a contract bug must stay visible instead of being hidden behind
+ * a fallback to another provider.
+ */
+export function isTransportError(error: unknown): boolean {
+	if (typeof error !== "object" || error === null) return false;
+
+	const named = error as {
+		name?: unknown;
+		code?: unknown;
+		cause?: unknown;
+		errors?: unknown;
+	};
+	if (typeof named.name === "string" && TRANSPORT_ERROR_NAMES.has(named.name)) return true;
+	if (typeof named.code === "string" && isTransportCode(named.code)) return true;
+
+	// An AggregateError collects the failures of several addresses (undici
+	// tries them one after the other) - one of them being a transport
+	// failure makes the whole attempt a transport failure.
+	if (Array.isArray(named.errors) && named.errors.some((entry) => isTransportError(entry))) {
+		return true;
+	}
+
+	// Node/undici wrap the real cause (e.g. ECONNREFUSED) one level deeper.
+	if (named.cause !== undefined && named.cause !== error) {
+		return isTransportError(named.cause);
+	}
+	return false;
+}
+
+/**
+ * True when the error code describes a transport failure.
+ */
+function isTransportCode(code: string): boolean {
+	if (TRANSPORT_ERROR_CODES.has(code)) return true;
+	return TRANSPORT_ERROR_CODE_PREFIXES.some((prefix) => code.startsWith(prefix));
+}
+
+/**
+ * Wrap any thrown value into a JevProviderError.
+ *
+ * A failure with an HTTP status is retryable when the status says so (see
+ * `isRetryableStatus`). A failure WITHOUT a status is only retryable when it
+ * is a known transport failure (network, timeout, abort) - an unknown error
+ * is most likely a bug in our own code and must not disappear behind a
+ * fallback to another provider.
+ */
+export function toProviderError(error: unknown, provider: JevProviderName): JevProviderError {
+	if (error instanceof JevProviderError) return error;
+	const status = extractStatus(error);
+	const message = error instanceof Error ? error.message : String(error);
+	return new JevProviderError(`${provider} provider failed: ${message}`, {
+		provider,
+		status,
+		retryable: status === undefined ? isTransportError(error) : isRetryableStatus(status),
+		cause: error,
+	});
+}

package/packages/jev-judge/src/task-intake.ts

@@ -62,8 +62,9 @@ export class TaskIntakeController {
 			...config,
 		} as Required<TaskIntakeConfig>;
 
-		// Initialize Jev if API key is available
-		if (config.apiKey) {
+		// Initialize Jev when a provider is available (an explicit provider
+		// chain or a legacy single key)
+		if ((config.providers && config.providers.length > 0) || config.apiKey) {
 			this.jev = new JevJudge(config as JevJudgeConfig);
 		}
 	}

package/packages/jev-judge/src/types.ts

@@ -18,15 +18,42 @@ export interface JevNoulQuestion {
 export interface JevChoiceQuestion<T extends string = string> {
   type: "choice";
   instructions: string;
-  options: T[];
+  /**
+   * Candidate names.
+   *
+   * The TypeSafe API does not know about `options`: it expects `criteria` as a
+   * DICTIONARY of `name -> description` (a list is rejected by the API with
+   * 422 "Input should be a valid dictionary"). `options` stays the ergonomic
+   * way to name the candidates.
+   */
+  options?: T[];
+  /**
+   * Choice criteria. A dictionary is used as-is; a list of names is converted
+   * to a dictionary whose description is the name itself. Takes precedence
+   * over `options`.
+   */
+  criteria?: Record<string, string> | string[];
+  /** Optional description per option name (used together with `options`). */
+  optionDescriptions?: Record<string, string>;
 }
 
-// Score: Rate something 0-100
+// Score: Rate something on a scale
 export interface JevScoreQuestion {
   type: "score";
   instructions: string;
+  /** Lower bound of the reported scale (default 0). */
   min?: number;
+  /** Upper bound of the reported scale (default 100). */
   max?: number;
+  /**
+   * Ordered rubric levels, lowest first.
+   *
+   * The TypeSafe API requires a rubric for score questions: the answer is the
+   * probability-weighted POSITION in this list (0 .. levels-1), never an
+   * arbitrary number. `min`/`max` only describe the scale the answer is
+   * reported on; when no rubric is given one is derived from them.
+   */
+  criteria?: string[];
 }
 
 // Any question type
@@ -76,6 +103,8 @@ export interface JevEvaluationResult<T extends string = string> {
   state: string;
   latencyMs: number;
   model: string;
+  /** The provider that served the request (when a provider chain is used). */
+  provider?: JevProviderName;
 }
 
 // Threshold-based action decision
@@ -86,11 +115,39 @@ export interface ThresholdDecision {
   action: "proceed" | "review" | "block";
 }
 
+// Provider identity
+export type JevProviderName = "typesafe" | "openrouter";
+
+/**
+ * A provider bound to its OWN credential.
+ *
+ * Credentials are never shared between providers: a TypeSafe key is only ever
+ * sent to the TypeSafe API and an OpenRouter key only to OpenRouter.
+ */
+export interface JevProviderConfig {
+  provider: JevProviderName;
+  apiKey: string;
+  baseUrl: string;
+  model: string;
+}
+
 // Config
 export interface JevJudgeConfig {
-  apiKey: string;
-  baseUrl?: string; // Defaults to OpenRouter
-  model?: string; // Defaults to typesafe/jev-1.13
+  /**
+   * Legacy single-provider key. Prefer `providers`: a bare key cannot be
+   * bound to a provider, which is how an OpenRouter key used to end up being
+   * sent to the TypeSafe API.
+   */
+  apiKey?: string;
+  baseUrl?: string; // Defaults to the TypeSafe API
+  model?: string; // Defaults to the provider's own model
+  /**
+   * Explicit provider chain, tried in order. A provider is only used for the
+   * next attempt when the previous one failed with a retryable error
+   * (network, timeout, rate limit, auth, 5xx) - never for a request contract
+   * error, which is a bug that must stay visible.
+   */
+  providers?: JevProviderConfig[];
   defaultThreshold?: number; // Default 0.7
   timeoutMs?: number;
 }