terramend 0.2.0

package/src/utils/providerErrors.test.ts

@@ -0,0 +1,315 @@
+import {
+  detectProviderError,
+  extractProviderId,
+  findProviderErrorMatch,
+  isProviderBillingExhausted,
+  isRouterKeylimitExhaustedError,
+} from "#app/utils/providerErrors";
+
+describe("detectProviderError", () => {
+  describe("false positives previously seen in production", () => {
+    it("returns null for commit SHAs containing 429", () => {
+      expect(detectProviderError("hash=7a46d89f505b36df49b4f54429daffa1a9459b11")).toBeNull();
+      expect(detectProviderError("commit f609cc89e84596ab125d60dac568bfb2ef398396 429")).toBeNull();
+    });
+
+    it("classifies 401 + x-ratelimit-* headers as auth, not rate-limited", () => {
+      // OpenRouter 401 responses bundle `x-ratelimit-*` rate-limit headers
+      // alongside the auth error. the auth patterns must win — pre-fix this
+      // got tagged as `rate limited` because of the loose `\brate[_ ]limit`
+      // match against header names like `ratelimit-limit-requests`. note: in
+      // OpenRouter's actual format the header name is `ratelimit` (one word),
+      // but the dumped JSON sometimes contains `rate-limit` separators too.
+      const stderr = JSON.stringify({
+        error: { name: "APIError", statusCode: 401, message: "Invalid authentication credentials" },
+        headers: {
+          "x-ratelimit-limit-requests": 50,
+          "x-ratelimit-remaining-requests": 49,
+          "x-ratelimit-reset-tokens": "2025-01-01T00:00:00Z",
+        },
+      });
+      expect(detectProviderError(stderr)).toBe("auth error (401)");
+    });
+
+    it("returns null for INTERNAL_SERVER_ERROR substring", () => {
+      expect(detectProviderError("HTTP/1.1 500 INTERNAL_SERVER_ERROR")).toBeNull();
+      expect(detectProviderError("expected: not INTERNAL_SERVER_ERROR")).toBeNull();
+    });
+
+    it("returns null for INTERNALS substring", () => {
+      expect(detectProviderError("debugging INTERNALS of the parser")).toBeNull();
+    });
+  });
+
+  describe("auth errors", () => {
+    it("detects 401 / 403 status codes as auth errors", () => {
+      expect(detectProviderError('{"statusCode": 401}')).toBe("auth error (401)");
+      expect(detectProviderError('{"statusCode": 403}')).toBe("auth error (403)");
+      expect(detectProviderError("status_code: 401")).toBe("auth error (401)");
+    });
+
+    it("detects OpenRouter 'User not found' (disabled/invalid key)", () => {
+      // bare `"code":401` lacks a status-key prefix so the 401 status pattern
+      // intentionally doesn't fire; the User-not-found pattern catches it.
+      expect(detectProviderError('{"error":{"message":"User not found","code":401}}')).toBe(
+        "auth error (invalid/disabled key)",
+      );
+      expect(detectProviderError("APIError: User not found.")).toBe(
+        "auth error (invalid/disabled key)",
+      );
+    });
+
+    it("detects 'Invalid authentication' phrasing", () => {
+      expect(detectProviderError("Invalid authentication credentials")).toBe(
+        "auth error (invalid credentials)",
+      );
+    });
+
+    it("detects 'No auth credentials found' phrasing", () => {
+      expect(detectProviderError("AI_APICallError: No auth credentials found")).toBe(
+        "auth error (missing credentials)",
+      );
+    });
+  });
+
+  describe("billing exhaustion", () => {
+    // see #778 — providers return 401 / 429 for billing/quota exhaustion
+    // (OpenCode Zen `CreditsError` / `FreeUsageLimitError`, Gemini
+    // `RESOURCE_EXHAUSTED` + spending cap, "Insufficient balance"). these
+    // are non-retryable; status-code patterns must NOT win and surface the
+    // misleading "auth error (401)" / "rate limited (429)" labels.
+    it("classifies OpenCode Zen CreditsError as billing exhausted, not 401", () => {
+      const stderr = JSON.stringify({
+        statusCode: 401,
+        responseBody:
+          '{"type":"error","error":{"type":"CreditsError","message":"Insufficient balance. Manage your billing here: https://opencode.ai/workspace/x/billing"}}',
+      });
+      expect(detectProviderError(stderr)).toBe("provider billing exhausted");
+    });
+
+    it("classifies OpenCode Zen FreeUsageLimitError as billing exhausted, not 429", () => {
+      const stderr = JSON.stringify({
+        statusCode: 429,
+        responseBody:
+          '{"type":"error","error":{"type":"FreeUsageLimitError","message":"Rate limit exceeded. Please try again later."}}',
+      });
+      expect(detectProviderError(stderr)).toBe("provider billing exhausted");
+    });
+
+    it("classifies Gemini spending-cap RESOURCE_EXHAUSTED as billing exhausted, not 429", () => {
+      const stderr =
+        'statusCode: 429, body: {"code": 429, "status": "RESOURCE_EXHAUSTED", "message": "Your project has exceeded its monthly spending cap..."}';
+      expect(detectProviderError(stderr)).toBe("provider billing exhausted");
+    });
+
+    it("classifies bare 'Insufficient balance' as billing exhausted", () => {
+      expect(detectProviderError("error: Insufficient balance")).toBe("provider billing exhausted");
+    });
+
+    it("classifies Anthropic 'credit balance is too low' as billing exhausted (#835)", () => {
+      // Anthropic-direct BYOK returns this string verbatim when the user's
+      // Anthropic console credit balance can't cover the request. distinct
+      // wording from "Insufficient balance" used by DeepSeek / OpenCode Zen.
+      const stderr =
+        "APIError: 400 Your credit balance is too low to access the Anthropic API. Please go to Plans & Billing to upgrade or purchase credits.";
+      expect(detectProviderError(stderr)).toBe("provider billing exhausted");
+    });
+  });
+
+  describe("real provider errors", () => {
+    it("detects 429 only when adjacent to a status key", () => {
+      expect(detectProviderError('{"statusCode": 429}')).toBe("rate limited (429)");
+      expect(detectProviderError('{"status_code": 429, "message": "..."}')).toBe(
+        "rate limited (429)",
+      );
+      expect(detectProviderError("http_status: 429")).toBe("rate limited (429)");
+      expect(detectProviderError("status=429")).toBe("rate limited (429)");
+    });
+
+    it("detects rate_limit_error and rate_limit_exceeded", () => {
+      expect(detectProviderError('{"type":"rate_limit_error"}')).toBe("rate limited");
+      expect(detectProviderError("rate_limit_exceeded")).toBe("rate limited");
+      expect(detectProviderError("plain rate limit reached")).toBe("rate limited");
+    });
+
+    it("detects rate-limit phrasing with trailing inflection", () => {
+      expect(detectProviderError("Error: rate limited by provider")).toBe("rate limited");
+      expect(detectProviderError("rate limits exceeded for this key")).toBe("rate limited");
+    });
+
+    it("detects RESOURCE_EXHAUSTED", () => {
+      expect(detectProviderError('"status": "RESOURCE_EXHAUSTED"')).toBe("quota exhausted");
+    });
+
+    it("detects gRPC INTERNAL status as a whole word", () => {
+      expect(detectProviderError('"status": "INTERNAL"')).toBe("provider internal error");
+    });
+
+    it("detects UNAVAILABLE as a whole word", () => {
+      expect(detectProviderError('"status": "UNAVAILABLE"')).toBe("provider unavailable");
+    });
+
+    it("detects 500 / 503 only when adjacent to a status key", () => {
+      expect(detectProviderError('"statusCode": 500')).toBe("provider 500 error");
+      expect(detectProviderError('"statusCode": 503')).toBe("provider unavailable (503)");
+      expect(detectProviderError("v1.503.0 release notes")).toBeNull();
+    });
+
+    it("detects quota and zero-quota responses", () => {
+      expect(detectProviderError('"message": "quota exceeded"')).toBe("quota error");
+      expect(detectProviderError('{"code":"insufficient_quota"}')).toBe("quota error");
+      expect(detectProviderError('"error":"quota_exceeded"')).toBe("quota error");
+      expect(detectProviderError('{"reason":"quotaExceeded"}')).toBe("quota error");
+      expect(detectProviderError('{"limit": 0, "remaining": 0}')).toBe("zero quota");
+      expect(detectProviderError('"time_limit": 0')).toBeNull();
+    });
+  });
+});
+
+describe("findProviderErrorMatch", () => {
+  // regression for issue #703: when stderr arrives as a multi-KB buffer
+  // (mcp tool-schema dump + the actual error message), the old
+  // `chunk.substring(0, 500)` excerpt showed the head of the buffer
+  // (schema) instead of the matched error text. the windowed excerpt
+  // must center on the matched line.
+  it("excerpt centers on the matched line, not the head of the buffer", () => {
+    const schemaDump =
+      "{".repeat(2000) +
+      '"name":"terramend_create_pull_request_review","description":"Submit a review..."';
+    const errorLine = "ERROR 2026-05-13 service=session error=rate_limit_exceeded retry-after=30";
+    const chunk = `${schemaDump}\n${errorLine}\ncaller stack at handler.ts:42`;
+
+    const match = findProviderErrorMatch(chunk);
+    expect(match).not.toBeNull();
+    expect(match?.label).toBe("rate limited");
+    expect(match?.excerpt).toContain("rate_limit_exceeded");
+    expect(match?.excerpt).toContain("retry-after=30");
+    expect(match?.excerpt).not.toContain("terramend_create_pull_request_review");
+  });
+
+  it("includes a small surrounding-line window for stack-trace context", () => {
+    const chunk =
+      "» about to call session.processor\n" +
+      "ERROR rate_limit_exceeded for key=abc\n" +
+      "at handler.ts:42\n" +
+      "at runtime.ts:88";
+    const match = findProviderErrorMatch(chunk);
+    expect(match?.excerpt).toContain("about to call session.processor");
+    expect(match?.excerpt).toContain("rate_limit_exceeded");
+    expect(match?.excerpt).toContain("handler.ts:42");
+    expect(match?.excerpt).toContain("runtime.ts:88");
+  });
+
+  it("falls back to the matched line alone when adjacent lines are huge", () => {
+    const giantPrefix = "x".repeat(5000);
+    const errorLine = '"statusCode": 429, "message": "slow down"';
+    const giantSuffix = "y".repeat(5000);
+    const chunk = `${giantPrefix}\n${errorLine}\n${giantSuffix}`;
+
+    const match = findProviderErrorMatch(chunk);
+    expect(match?.label).toBe("rate limited (429)");
+    expect(match?.excerpt).toBe(errorLine);
+  });
+
+  it("head-truncates the matched line if it alone exceeds the byte cap", () => {
+    const padding = "z".repeat(700);
+    const chunk = `${padding} "statusCode": 429 ${padding}`;
+    const match = findProviderErrorMatch(chunk);
+    expect(match?.label).toBe("rate limited (429)");
+    expect(match?.excerpt.length).toBeLessThanOrEqual(600);
+  });
+
+  it("returns null when no pattern matches", () => {
+    expect(findProviderErrorMatch("just some normal log line\nnothing wrong here")).toBeNull();
+  });
+});
+
+describe("isProviderBillingExhausted (#835)", () => {
+  it("matches DeepSeek 'Insufficient Balance' payloads", () => {
+    expect(isProviderBillingExhausted("AI_APICallError: Insufficient Balance")).toBe(true);
+  });
+
+  it("matches Anthropic 'credit balance is too low' payloads", () => {
+    expect(
+      isProviderBillingExhausted("Your credit balance is too low to access the Anthropic API"),
+    ).toBe(true);
+  });
+
+  it("matches OpenCode Zen CreditsError / FreeUsageLimitError", () => {
+    expect(isProviderBillingExhausted("CreditsError: out of credit")).toBe(true);
+    expect(isProviderBillingExhausted("FreeUsageLimitError: limit hit")).toBe(true);
+  });
+
+  it("returns false for unrelated provider errors", () => {
+    expect(isProviderBillingExhausted('{"statusCode": 401}')).toBe(false);
+    expect(isProviderBillingExhausted("rate_limit_exceeded")).toBe(false);
+    expect(isProviderBillingExhausted("just some log noise")).toBe(false);
+  });
+});
+
+describe("extractProviderId", () => {
+  it("parses providerID= from OpenCode harness logs", () => {
+    expect(
+      extractProviderId(
+        'ERROR providerID=deepseek modelID=deepseek-v4-pro error={"name":"AI_APICallError"}',
+      ),
+    ).toBe("deepseek");
+  });
+
+  it("lowercases the captured slug", () => {
+    expect(extractProviderId("providerID=Anthropic modelID=claude")).toBe("anthropic");
+  });
+
+  it("returns null when providerID is absent", () => {
+    expect(extractProviderId("APIError: Insufficient Balance")).toBeNull();
+  });
+});
+
+describe("isRouterKeylimitExhaustedError", () => {
+  it("matches the canonical OpenRouter mid-run error", () => {
+    expect(
+      isRouterKeylimitExhaustedError(
+        "APIError: This request requires more credits, or fewer max_tokens. " +
+          "You requested up to 32000 tokens, but can only afford 22800. " +
+          "To increase, visit https://openrouter.ai/settings/keys and create a key with a higher total limit",
+      ),
+    ).toBe(true);
+  });
+
+  it("matches the 'requires more credits' phrasing on its own", () => {
+    expect(
+      isRouterKeylimitExhaustedError("This request requires more credits, or fewer max_tokens."),
+    ).toBe(true);
+  });
+
+  it("matches the 'requested up to ... can only afford' phrasing on its own", () => {
+    expect(
+      isRouterKeylimitExhaustedError("You requested up to 8000 tokens but can only afford 1234"),
+    ).toBe(true);
+  });
+
+  it("does not match generic out-of-credit text", () => {
+    expect(isRouterKeylimitExhaustedError("Your account has insufficient credits")).toBe(false);
+    expect(isRouterKeylimitExhaustedError("rate_limit_exceeded")).toBe(false);
+    expect(isRouterKeylimitExhaustedError('{"limit": 0}')).toBe(false);
+  });
+
+  it("does not match unrelated mentions of max_tokens", () => {
+    expect(isRouterKeylimitExhaustedError("max_tokens parameter must be a positive integer")).toBe(
+      false,
+    );
+  });
+
+  it("matches across newlines (defends against upstream wrapping/reformatting)", () => {
+    expect(
+      isRouterKeylimitExhaustedError(
+        "APIError: This request requires more credits, or\nfewer max_tokens. You requested up to 32000 tokens",
+      ),
+    ).toBe(true);
+    expect(
+      isRouterKeylimitExhaustedError(
+        "You requested up to 32000 tokens,\nbut can only afford 22800",
+      ),
+    ).toBe(true);
+  });
+});

package/src/utils/providerErrors.ts

@@ -0,0 +1,172 @@
+type ProviderErrorPattern = { regex: RegExp; label: string };
+
+/** Stable label for the BYOK provider-billing-exhausted classification. */
+export const PROVIDER_BILLING_EXHAUSTED_LABEL = "provider billing exhausted";
+
+// status codes are only treated as provider errors when they are adjacent to
+// a recognised status key. this rejects commit SHAs that happen to contain
+// "429", version strings, file hashes, etc.
+const statusKey = `\\b(?:status[_ ]?code|http[_ ]?status|status)["']?\\s*[:=]\\s*["']?`;
+
+const PROVIDER_ERROR_PATTERNS: ProviderErrorPattern[] = [
+  // billing-payload patterns come BEFORE bare status-code patterns. providers
+  // commonly return 401 / 429 for billing/quota exhaustion (OpenCode Zen
+  // `CreditsError` / `FreeUsageLimitError`, Gemini `RESOURCE_EXHAUSTED` +
+  // "spending cap", Anthropic "Insufficient balance" / "credit balance is
+  // too low"). these are non-retryable and require user-billing action —
+  // distinct from a transient auth error or rate-limit. status-code patterns
+  // would otherwise win and surface "auth error (401)" / "rate limited (429)"
+  // with no billing hint. see #778, #835.
+  { regex: /\bCreditsError\b/, label: PROVIDER_BILLING_EXHAUSTED_LABEL },
+  { regex: /\bFreeUsageLimitError\b/, label: PROVIDER_BILLING_EXHAUSTED_LABEL },
+  { regex: /Insufficient balance/i, label: PROVIDER_BILLING_EXHAUSTED_LABEL },
+  { regex: /credit balance is too low/i, label: PROVIDER_BILLING_EXHAUSTED_LABEL },
+  { regex: /spending cap/i, label: PROVIDER_BILLING_EXHAUSTED_LABEL },
+  // auth patterns must come BEFORE rate-limit patterns. OpenRouter 401 error
+  // payloads carry `x-ratelimit-*` response headers in the dump, and the
+  // free-form rate-limit regex below would otherwise win on word-boundary
+  // matches inside header names. canonical 401 messages: OpenRouter returns
+  // `{"error":{"message":"User not found","code":401}}` for disabled or
+  // invalid keys (https://openai.luzhipeng.com/docs/api/reference/errors-and-debugging).
+  { regex: new RegExp(`${statusKey}401\\b`, "i"), label: "auth error (401)" },
+  { regex: new RegExp(`${statusKey}403\\b`, "i"), label: "auth error (403)" },
+  { regex: /\bUser not found\b/i, label: "auth error (invalid/disabled key)" },
+  { regex: /\bInvalid authentication\b/i, label: "auth error (invalid credentials)" },
+  { regex: /\bNo auth credentials found\b/i, label: "auth error (missing credentials)" },
+  { regex: new RegExp(`${statusKey}429\\b`, "i"), label: "rate limited (429)" },
+  { regex: new RegExp(`${statusKey}500\\b`, "i"), label: "provider 500 error" },
+  { regex: new RegExp(`${statusKey}503\\b`, "i"), label: "provider unavailable (503)" },
+  // matches `rate limit`, `rate limited`, `rate limits exceeded`,
+  // `rate_limit_error`, `rate_limit_exceeded`. the leading `\b` + `[_ ]`
+  // separator rejects `x-ratelimit-*` / `anthropic-ratelimit-*` response
+  // headers (no separator between "rate" and "limit") which routinely
+  // appear in dumped 401 / 4xx error JSON.
+  { regex: /\brate[_ ]limit/i, label: "rate limited" },
+  { regex: /\bRESOURCE_EXHAUSTED\b/, label: "quota exhausted" },
+  // Google gRPC `INTERNAL` status. word-boundary anchors reject
+  // `INTERNAL_SERVER_ERROR` (HTTP 500 message that may appear in unrelated
+  // log lines) and identifiers like `INTERNALS`.
+  { regex: /\bINTERNAL\b/, label: "provider internal error" },
+  { regex: /\bUNAVAILABLE\b/, label: "provider unavailable" },
+  // matches `quota`, `insufficient_quota`, `quota_exceeded`, `quotaExceeded`.
+  // word-character lookarounds would reject `_quota` / `quotaX`; `quota` is
+  // specific enough that a plain substring match is safe.
+  { regex: /quota/i, label: "quota error" },
+  // explicit zero-quota response, e.g. `{"limit": 0}`. the `\b` anchor
+  // around `limit` rejects keys like `time_limit` or `field_limit`.
+  { regex: /["']?\blimit\b["']?\s*:\s*0\b/, label: "zero quota" },
+];
+
+/**
+ * Result of a provider-error scan: the classification label plus a
+ * human-readable excerpt centered on the matched line. The excerpt is what
+ * gets surfaced in `» provider error detected (...)` log lines — see
+ * `extractExcerpt` for the windowing/byte-cap policy.
+ */
+export type ProviderErrorMatch = {
+  label: string;
+  excerpt: string;
+};
+
+// roughly half a wide terminal line by 4–5 lines of context; large enough
+// to capture a structured error payload (request id, retry-after, model)
+// plus its immediate stack/headers, small enough to not flood the log.
+const EXCERPT_MAX_BYTES = 600;
+const LINES_BEFORE = 1;
+const LINES_AFTER = 2;
+
+export function findProviderErrorMatch(text: string): ProviderErrorMatch | null {
+  for (const entry of PROVIDER_ERROR_PATTERNS) {
+    const m = entry.regex.exec(text);
+    if (!m) continue;
+    return { label: entry.label, excerpt: extractExcerpt(text, m.index) };
+  }
+  return null;
+}
+
+export function detectProviderError(text: string): string | null {
+  return findProviderErrorMatch(text)?.label ?? null;
+}
+
+/**
+ * Slice a context window around `matchIndex`: the matched line plus
+ * `LINES_BEFORE`/`LINES_AFTER` neighbours. If the windowed slice exceeds
+ * `EXCERPT_MAX_BYTES` (giant adjacent lines, e.g. JSON tool-schema dumps),
+ * fall back to the matched line alone, head-truncated if still too long.
+ * Replaces the old `chunk.substring(0, 500)` head-anchored excerpt which
+ * surfaced whatever happened to be at the front of the stderr buffer
+ * instead of the error itself. See issue #703.
+ */
+function extractExcerpt(text: string, matchIndex: number): string {
+  const lineStart = text.lastIndexOf("\n", matchIndex - 1) + 1;
+  const lineEndRaw = text.indexOf("\n", matchIndex);
+  const lineEnd = lineEndRaw === -1 ? text.length : lineEndRaw;
+
+  let start = lineStart;
+  for (let i = 0; i < LINES_BEFORE && start > 0; i++) {
+    const prev = text.lastIndexOf("\n", start - 2);
+    start = prev < 0 ? 0 : prev + 1;
+  }
+
+  let end = lineEnd;
+  for (let i = 0; i < LINES_AFTER && end < text.length; i++) {
+    const next = text.indexOf("\n", end + 1);
+    end = next < 0 ? text.length : next;
+  }
+
+  let excerpt = text.slice(start, end);
+  if (excerpt.length > EXCERPT_MAX_BYTES) {
+    excerpt = text.slice(lineStart, lineEnd);
+    if (excerpt.length > EXCERPT_MAX_BYTES) excerpt = excerpt.slice(0, EXCERPT_MAX_BYTES);
+  }
+  return excerpt.trim();
+}
+
+/**
+ * OpenRouter's response when the per-run key's remaining budget can't cover
+ * the agent's `max_tokens` reservation. Distinct from a generic provider error
+ * because it's a Terramend billing concern, not an upstream outage — the user's
+ * Router wallet ran out (or the key budget was undersized at mint time and the
+ * agent ran out of headroom partway through).
+ *
+ * Match must be specific to this exact OpenRouter error class. Generic "credits"
+ * or "limit" text shows up in unrelated errors and would mis-classify them.
+ *
+ * Sample:
+ *   `APIError: This request requires more credits, or fewer max_tokens.
+ *    You requested up to 32000 tokens, but can only afford 22800.`
+ */
+// `/s` (dotAll) lets `.*?` cross newlines so we still detect the error if any
+// upstream layer reformats the message onto multiple lines. Without it, a
+// single inserted `\n` would silently bypass the BillingError reclassification
+// and the user would see the generic `❌ Terramend failed` dump instead of the
+// actionable top-up CTA.
+const ROUTER_KEYLIMIT_EXHAUSTED_PATTERN =
+  /requires more credits.*?fewer max_tokens|requested up to \d+ tokens.*?can only afford/is;
+
+export function isRouterKeylimitExhaustedError(text: string): boolean {
+  return ROUTER_KEYLIMIT_EXHAUSTED_PATTERN.test(text);
+}
+
+/**
+ * BYOK billing-exhausted: provider rejected the request because the user's
+ * provider wallet is empty (DeepSeek "Insufficient Balance", Anthropic
+ * "credit balance is too low", OpenCode Zen `CreditsError` /
+ * `FreeUsageLimitError`, Gemini "spending cap"). Distinct from
+ * `isRouterKeylimitExhaustedError` — that's Terramend's Router wallet, this
+ * is the user's own provider account.
+ */
+export function isProviderBillingExhausted(text: string): boolean {
+  return findProviderErrorMatch(text)?.label === PROVIDER_BILLING_EXHAUSTED_LABEL;
+}
+
+/**
+ * Extract `providerID=foo` from agent error logs (OpenCode emits this on
+ * `provider error detected (...)` lines). Returns the lowercase provider
+ * slug, or null when absent. Used to render a provider-specific dashboard
+ * link in the BYOK billing-exhausted summary.
+ */
+export function extractProviderId(text: string): string | null {
+  const match = text.match(/\bproviderID=([a-z0-9_-]+)/i);
+  return match ? match[1]!.toLowerCase() : null;
+}