scrapebadger 0.24.1 → 0.25.0

package/dist/{index-CIZUd1Zr.d.cts → index-ZD3sVCjc.d.cts}

@@ -52,6 +52,22 @@ declare class BaseClient {
      * Make an HTTP request to the API.
      */
     request<T>(path: string, options?: RequestOptions): Promise<T>;
+    /**
+     * POST and return the undecoded response body.
+     *
+     * For endpoints that answer with something other than JSON — currently
+     * `/v1/web/scrape` with `raw_content: true`, which returns the scraped body
+     * itself. The normal path funnels a non-JSON response into
+     * `{ detail: await response.text() }`, which both loses the result and, for
+     * a binary payload, corrupts it: `text()` decodes bytes as UTF-8.
+     *
+     * Returns the raw bytes plus the response headers.
+     */
+    postBinary(path: string, options?: RequestOptions): Promise<{
+        bytes: Uint8Array;
+        headers: Headers;
+        status: number;
+    }>;
     /**
      * Make an HTTP request and return both data and rate limit headers.
      */
@@ -76,6 +92,14 @@ declare class BaseClient {
      * Handle HTTP response and convert errors.
      */
     private handleResponse;
+    /**
+     * Whether a failed request is worth another attempt.
+     *
+     * Retryable: transient server failures (500/502/503/504), request timeouts,
+     * rate limits, and raw network faults thrown by `fetch` itself. Everything
+     * else — auth, validation, not-found, conflict — is final.
+     */
+    private static isRetryable;
     /**
      * Sleep for a given duration.
      */

package/dist/{index-CIZUd1Zr.d.ts → index-ZD3sVCjc.d.ts}

@@ -52,6 +52,22 @@ declare class BaseClient {
      * Make an HTTP request to the API.
      */
     request<T>(path: string, options?: RequestOptions): Promise<T>;
+    /**
+     * POST and return the undecoded response body.
+     *
+     * For endpoints that answer with something other than JSON — currently
+     * `/v1/web/scrape` with `raw_content: true`, which returns the scraped body
+     * itself. The normal path funnels a non-JSON response into
+     * `{ detail: await response.text() }`, which both loses the result and, for
+     * a binary payload, corrupts it: `text()` decodes bytes as UTF-8.
+     *
+     * Returns the raw bytes plus the response headers.
+     */
+    postBinary(path: string, options?: RequestOptions): Promise<{
+        bytes: Uint8Array;
+        headers: Headers;
+        status: number;
+    }>;
     /**
      * Make an HTTP request and return both data and rate limit headers.
      */
@@ -76,6 +92,14 @@ declare class BaseClient {
      * Handle HTTP response and convert errors.
      */
     private handleResponse;
+    /**
+     * Whether a failed request is worth another attempt.
+     *
+     * Retryable: transient server failures (500/502/503/504), request timeouts,
+     * rate limits, and raw network faults thrown by `fetch` itself. Everything
+     * else — auth, validation, not-found, conflict — is final.
+     */
+    private static isRetryable;
     /**
      * Sleep for a given duration.
      */

package/dist/index.d.cts

@@ -1,5 +1,5 @@
-import { B as BaseClient, T as TwitterClient, S as ScrapeBadgerConfig } from './index-CIZUd1Zr.cjs';
-export { e as AccountRestrictedError, a3 as ApiResponse, $ as Article, A as AuthenticationError, aj as BillingLog, ak as BillingLogList, a2 as Broadcast, j as CommunitiesClient, K as Community, F as CommunityBanner, O as CommunityMember, a0 as CommunityNote, J as CommunityRule, q as CommunityTweetType, C as ConflictError, al as ConnectOptions, ab as ConnectedEvent, a8 as CreateMonitorParams, ah as DeliveryLog, ai as DeliveryLogList, ae as ErrorEvent, ao as FilterRuleCreate, at as FilterRuleDeliveryLog, au as FilterRuleDeliveryLogListResponse, ar as FilterRuleListResponse, aq as FilterRulePricingTier, av as FilterRulePricingTiersResponse, an as FilterRuleResponse, am as FilterRuleStatus, ap as FilterRuleUpdate, as as FilterRuleValidateResponse, G as GeoClient, l as GeoSearchOptions, H as Hashtag, I as InsufficientCreditsError, g as IteratorOptions, E as List, a4 as ListResponse, L as ListsClient, Y as Location, M as Media, a5 as MonitorStatus, N as NotFoundError, P as PaginatedResponse, f as PaginationOptions, ac as PingEvent, _ as Place, Z as PlaceTrends, s as Poll, r as PollOption, Q as QueryType, b as RateLimitError, R as ResolvedConfig, a as ScrapeBadgerError, c as ServerError, a1 as Space, o as SpacesClient, m as StreamClient, n as StreamEmitter, af as StreamEvent, ag as StreamEventType, a6 as StreamMonitor, a7 as StreamMonitorList, aa as StreamTweet, d as TimeoutError, X as Trend, p as TrendCategory, k as TrendsClient, x as Tweet, ad as TweetEvent, w as TweetPlace, i as TweetsClient, a9 as UpdateMonitorParams, t as Url, y as User, z as UserAbout, D as UserIds, u as UserMention, U as UsersClient, V as ValidationError, W as WebSocketStreamError, h as collectAll, v as verifyWebhookSignature } from './index-CIZUd1Zr.cjs';
+import { B as BaseClient, T as TwitterClient, S as ScrapeBadgerConfig } from './index-ZD3sVCjc.cjs';
+export { e as AccountRestrictedError, a3 as ApiResponse, $ as Article, A as AuthenticationError, aj as BillingLog, ak as BillingLogList, a2 as Broadcast, j as CommunitiesClient, K as Community, F as CommunityBanner, O as CommunityMember, a0 as CommunityNote, J as CommunityRule, q as CommunityTweetType, C as ConflictError, al as ConnectOptions, ab as ConnectedEvent, a8 as CreateMonitorParams, ah as DeliveryLog, ai as DeliveryLogList, ae as ErrorEvent, ao as FilterRuleCreate, at as FilterRuleDeliveryLog, au as FilterRuleDeliveryLogListResponse, ar as FilterRuleListResponse, aq as FilterRulePricingTier, av as FilterRulePricingTiersResponse, an as FilterRuleResponse, am as FilterRuleStatus, ap as FilterRuleUpdate, as as FilterRuleValidateResponse, G as GeoClient, l as GeoSearchOptions, H as Hashtag, I as InsufficientCreditsError, g as IteratorOptions, E as List, a4 as ListResponse, L as ListsClient, Y as Location, M as Media, a5 as MonitorStatus, N as NotFoundError, P as PaginatedResponse, f as PaginationOptions, ac as PingEvent, _ as Place, Z as PlaceTrends, s as Poll, r as PollOption, Q as QueryType, b as RateLimitError, R as ResolvedConfig, a as ScrapeBadgerError, c as ServerError, a1 as Space, o as SpacesClient, m as StreamClient, n as StreamEmitter, af as StreamEvent, ag as StreamEventType, a6 as StreamMonitor, a7 as StreamMonitorList, aa as StreamTweet, d as TimeoutError, X as Trend, p as TrendCategory, k as TrendsClient, x as Tweet, ad as TweetEvent, w as TweetPlace, i as TweetsClient, a9 as UpdateMonitorParams, t as Url, y as User, z as UserAbout, D as UserIds, u as UserMention, U as UsersClient, V as ValidationError, W as WebSocketStreamError, h as collectAll, v as verifyWebhookSignature } from './index-ZD3sVCjc.cjs';
 import 'node:events';
 
 /**
@@ -65,6 +65,22 @@ interface ScrapeResult {
     url: string;
     status_code: number;
     content: string | null;
+    /**
+     * Base64 body for a binary target (image/PDF/archive). Set instead of
+     * `content`, which is null there — binary bytes have no text form.
+     * Null when the body exceeded the 25 MB base64 ceiling.
+     */
+    content_base64: string | null;
+    /** Whether the target returned a binary (non-text) body. */
+    is_binary: boolean;
+    /** The target's response Content-Type, as a bare media type. */
+    content_type: string | null;
+    /**
+     * Undecoded response body. Only set when `rawContent: true` — that mode
+     * returns the body itself rather than a JSON envelope. Write it straight to
+     * a file; do not decode it, the payload may be an image or a PDF.
+     */
+    content_bytes?: Uint8Array;
     format: string;
     engine_used: string;
     credits_used: number;
@@ -144,6 +160,16 @@ declare class WebClient {
      * @returns The scrape result including content, metadata, and credit usage
      */
     scrape(url: string, options?: ScrapeOptions): Promise<ScrapeResult>;
+    /**
+     * Run a `rawContent` scrape, whose response is not JSON.
+     *
+     * The normal path funnels a non-JSON response into `{ detail: text }`, so a
+     * raw scrape returned a result with no content — and for a binary target,
+     * `response.text()` decoded the bytes as UTF-8 and destroyed them. Read the
+     * body as bytes and rebuild the metadata from the `X-Scrape-*` headers the
+     * server sends in this mode.
+     */
+    private scrapeRaw;
     /**
      * Extract structured data from a web page using AI.
      *

package/dist/index.d.ts

@@ -1,5 +1,5 @@
-import { B as BaseClient, T as TwitterClient, S as ScrapeBadgerConfig } from './index-CIZUd1Zr.js';
-export { e as AccountRestrictedError, a3 as ApiResponse, $ as Article, A as AuthenticationError, aj as BillingLog, ak as BillingLogList, a2 as Broadcast, j as CommunitiesClient, K as Community, F as CommunityBanner, O as CommunityMember, a0 as CommunityNote, J as CommunityRule, q as CommunityTweetType, C as ConflictError, al as ConnectOptions, ab as ConnectedEvent, a8 as CreateMonitorParams, ah as DeliveryLog, ai as DeliveryLogList, ae as ErrorEvent, ao as FilterRuleCreate, at as FilterRuleDeliveryLog, au as FilterRuleDeliveryLogListResponse, ar as FilterRuleListResponse, aq as FilterRulePricingTier, av as FilterRulePricingTiersResponse, an as FilterRuleResponse, am as FilterRuleStatus, ap as FilterRuleUpdate, as as FilterRuleValidateResponse, G as GeoClient, l as GeoSearchOptions, H as Hashtag, I as InsufficientCreditsError, g as IteratorOptions, E as List, a4 as ListResponse, L as ListsClient, Y as Location, M as Media, a5 as MonitorStatus, N as NotFoundError, P as PaginatedResponse, f as PaginationOptions, ac as PingEvent, _ as Place, Z as PlaceTrends, s as Poll, r as PollOption, Q as QueryType, b as RateLimitError, R as ResolvedConfig, a as ScrapeBadgerError, c as ServerError, a1 as Space, o as SpacesClient, m as StreamClient, n as StreamEmitter, af as StreamEvent, ag as StreamEventType, a6 as StreamMonitor, a7 as StreamMonitorList, aa as StreamTweet, d as TimeoutError, X as Trend, p as TrendCategory, k as TrendsClient, x as Tweet, ad as TweetEvent, w as TweetPlace, i as TweetsClient, a9 as UpdateMonitorParams, t as Url, y as User, z as UserAbout, D as UserIds, u as UserMention, U as UsersClient, V as ValidationError, W as WebSocketStreamError, h as collectAll, v as verifyWebhookSignature } from './index-CIZUd1Zr.js';
+import { B as BaseClient, T as TwitterClient, S as ScrapeBadgerConfig } from './index-ZD3sVCjc.js';
+export { e as AccountRestrictedError, a3 as ApiResponse, $ as Article, A as AuthenticationError, aj as BillingLog, ak as BillingLogList, a2 as Broadcast, j as CommunitiesClient, K as Community, F as CommunityBanner, O as CommunityMember, a0 as CommunityNote, J as CommunityRule, q as CommunityTweetType, C as ConflictError, al as ConnectOptions, ab as ConnectedEvent, a8 as CreateMonitorParams, ah as DeliveryLog, ai as DeliveryLogList, ae as ErrorEvent, ao as FilterRuleCreate, at as FilterRuleDeliveryLog, au as FilterRuleDeliveryLogListResponse, ar as FilterRuleListResponse, aq as FilterRulePricingTier, av as FilterRulePricingTiersResponse, an as FilterRuleResponse, am as FilterRuleStatus, ap as FilterRuleUpdate, as as FilterRuleValidateResponse, G as GeoClient, l as GeoSearchOptions, H as Hashtag, I as InsufficientCreditsError, g as IteratorOptions, E as List, a4 as ListResponse, L as ListsClient, Y as Location, M as Media, a5 as MonitorStatus, N as NotFoundError, P as PaginatedResponse, f as PaginationOptions, ac as PingEvent, _ as Place, Z as PlaceTrends, s as Poll, r as PollOption, Q as QueryType, b as RateLimitError, R as ResolvedConfig, a as ScrapeBadgerError, c as ServerError, a1 as Space, o as SpacesClient, m as StreamClient, n as StreamEmitter, af as StreamEvent, ag as StreamEventType, a6 as StreamMonitor, a7 as StreamMonitorList, aa as StreamTweet, d as TimeoutError, X as Trend, p as TrendCategory, k as TrendsClient, x as Tweet, ad as TweetEvent, w as TweetPlace, i as TweetsClient, a9 as UpdateMonitorParams, t as Url, y as User, z as UserAbout, D as UserIds, u as UserMention, U as UsersClient, V as ValidationError, W as WebSocketStreamError, h as collectAll, v as verifyWebhookSignature } from './index-ZD3sVCjc.js';
 import 'node:events';
 
 /**
@@ -65,6 +65,22 @@ interface ScrapeResult {
     url: string;
     status_code: number;
     content: string | null;
+    /**
+     * Base64 body for a binary target (image/PDF/archive). Set instead of
+     * `content`, which is null there — binary bytes have no text form.
+     * Null when the body exceeded the 25 MB base64 ceiling.
+     */
+    content_base64: string | null;
+    /** Whether the target returned a binary (non-text) body. */
+    is_binary: boolean;
+    /** The target's response Content-Type, as a bare media type. */
+    content_type: string | null;
+    /**
+     * Undecoded response body. Only set when `rawContent: true` — that mode
+     * returns the body itself rather than a JSON envelope. Write it straight to
+     * a file; do not decode it, the payload may be an image or a PDF.
+     */
+    content_bytes?: Uint8Array;
     format: string;
     engine_used: string;
     credits_used: number;
@@ -144,6 +160,16 @@ declare class WebClient {
      * @returns The scrape result including content, metadata, and credit usage
      */
     scrape(url: string, options?: ScrapeOptions): Promise<ScrapeResult>;
+    /**
+     * Run a `rawContent` scrape, whose response is not JSON.
+     *
+     * The normal path funnels a non-JSON response into `{ detail: text }`, so a
+     * raw scrape returned a result with no content — and for a binary target,
+     * `response.text()` decoded the bytes as UTF-8 and destroyed them. Read the
+     * body as bytes and rebuild the metadata from the `X-Scrape-*` headers the
+     * server sends in this mode.
+     */
+    private scrapeRaw;
     /**
      * Extract structured data from a web page using AI.
      *

package/dist/index.js

@@ -9,7 +9,7 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
 var WebSocket__default = /*#__PURE__*/_interopDefault(WebSocket);
 
 // src/internal/version.ts
-var SDK_VERSION = "0.24.1";
+var SDK_VERSION = "0.24.2";
 
 // src/internal/exceptions.ts
 var ScrapeBadgerError = class _ScrapeBadgerError extends Error {
@@ -124,7 +124,8 @@ var WebSocketStreamError = class _WebSocketStreamError extends ScrapeBadgerError
 };
 
 // src/internal/client.ts
-var BaseClient = class {
+var RETRYABLE_STATUS_CODES = [500, 502, 503, 504];
+var BaseClient = class _BaseClient {
   config;
   constructor(config) {
     this.config = config;
@@ -136,6 +137,39 @@ var BaseClient = class {
     const { data } = await this.requestRaw(path, options);
     return data;
   }
+  /**
+   * POST and return the undecoded response body.
+   *
+   * For endpoints that answer with something other than JSON — currently
+   * `/v1/web/scrape` with `raw_content: true`, which returns the scraped body
+   * itself. The normal path funnels a non-JSON response into
+   * `{ detail: await response.text() }`, which both loses the result and, for
+   * a binary payload, corrupts it: `text()` decodes bytes as UTF-8.
+   *
+   * Returns the raw bytes plus the response headers.
+   */
+  async postBinary(path, options = {}) {
+    const url = new URL(path, this.config.baseUrl);
+    const { body, headers = {} } = options;
+    const response = await this.fetchWithTimeout(url.toString(), {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        "X-API-Key": this.config.apiKey,
+        "User-Agent": `scrapebadger-node/${SDK_VERSION}`,
+        ...headers
+      },
+      body: body ? JSON.stringify(body) : void 0
+    });
+    if (!response.ok) {
+      await this.handleResponse(response);
+    }
+    return {
+      bytes: new Uint8Array(await response.arrayBuffer()),
+      headers: response.headers,
+      status: response.status
+    };
+  }
   /**
    * Make an HTTP request and return both data and rate limit headers.
    */
@@ -184,7 +218,7 @@ var BaseClient = class {
         return { data, rateLimit };
       } catch (error) {
         lastError = error;
-        if (error instanceof ScrapeBadgerError && !(error instanceof RateLimitError)) {
+        if (!_BaseClient.isRetryable(error)) {
           throw error;
         }
         if (attempt === this.config.maxRetries) {
@@ -311,6 +345,22 @@ var BaseClient = class {
         throw new ScrapeBadgerError(message);
     }
   }
+  /**
+   * Whether a failed request is worth another attempt.
+   *
+   * Retryable: transient server failures (500/502/503/504), request timeouts,
+   * rate limits, and raw network faults thrown by `fetch` itself. Everything
+   * else — auth, validation, not-found, conflict — is final.
+   */
+  static isRetryable(error) {
+    if (error instanceof ServerError) {
+      return RETRYABLE_STATUS_CODES.includes(error.statusCode);
+    }
+    if (error instanceof RateLimitError || error instanceof TimeoutError) {
+      return true;
+    }
+    return !(error instanceof ScrapeBadgerError);
+  }
   /**
    * Sleep for a given duration.
    */
@@ -2464,11 +2514,54 @@ var WebClient = class {
     if (options.aiPrompt !== void 0) body.ai_prompt = options.aiPrompt;
     if (options.rawContent !== void 0) body.raw_content = options.rawContent;
     if (options.skipBotDetection !== void 0) body.skip_bot_detection = options.skipBotDetection;
+    if (options.rawContent) {
+      return this.scrapeRaw(body);
+    }
     return this.client.request("/v1/web/scrape", {
       method: "POST",
       body
     });
   }
+  /**
+   * Run a `rawContent` scrape, whose response is not JSON.
+   *
+   * The normal path funnels a non-JSON response into `{ detail: text }`, so a
+   * raw scrape returned a result with no content — and for a binary target,
+   * `response.text()` decoded the bytes as UTF-8 and destroyed them. Read the
+   * body as bytes and rebuild the metadata from the `X-Scrape-*` headers the
+   * server sends in this mode.
+   */
+  async scrapeRaw(body) {
+    const { bytes, headers, status } = await this.client.postBinary("/v1/web/scrape", {
+      body
+    });
+    const int = (name) => {
+      const parsed = Number.parseInt(headers.get(name) ?? "", 10);
+      return Number.isNaN(parsed) ? 0 : parsed;
+    };
+    const mediaType = ((headers.get("content-type") ?? "").split(";")[0] ?? "").trim().toLowerCase();
+    const isText = mediaType.startsWith("text/") || ["application/json", "application/xml", "image/svg+xml"].includes(mediaType);
+    return {
+      success: headers.get("x-scrape-success") !== "0",
+      url: headers.get("x-scrape-url") ?? (typeof body.url === "string" ? body.url : ""),
+      status_code: int("x-scrape-status-code") || status,
+      content: isText ? new TextDecoder().decode(bytes) : null,
+      content_bytes: bytes,
+      content_base64: null,
+      is_binary: !isText,
+      content_type: mediaType || null,
+      format: headers.get("x-scrape-format") ?? "html",
+      engine_used: headers.get("x-scrape-engine") ?? "",
+      credits_used: int("x-credits-used"),
+      duration_ms: int("x-scrape-duration-ms"),
+      retries_used: int("x-scrape-retries"),
+      content_length: int("x-scrape-content-length") || bytes.length,
+      screenshot_url: null,
+      video_url: null,
+      headers: {},
+      blocking_detected: false
+    };
+  }
   /**
    * Extract structured data from a web page using AI.
    *