scrapebadger 0.3.0 → 0.4.0

package/README.md

@@ -19,7 +19,8 @@ The official Node.js/TypeScript client library for the [ScrapeBadger](https://sc
 - **Full TypeScript Support** - Complete type definitions for all API endpoints
 - **Modern ESM & CommonJS** - Works with both module systems
 - **Async Iterators** - Automatic pagination with `for await...of` syntax
-- **Retry Logic** - Built-in exponential backoff for reliability
+- **Smart Rate Limit Handling** - Reads API rate limit headers and automatically throttles pagination to avoid hitting limits
+- **Resilient Retries** - 10 automatic retries with exponential backoff on server errors, with colored console warnings on each retry
 - **Error Handling** - Typed exceptions for different error scenarios
 - **Tree Shakeable** - Import only what you need
 
@@ -345,20 +346,45 @@ const client = new ScrapeBadger({
   // Required: Your API key (or use SCRAPEBADGER_API_KEY env var)
   apiKey: "your-api-key",
 
-  // Optional: Custom base URL (default: https://api.scrapebadger.com)
-  baseUrl: "https://api.scrapebadger.com",
+  // Optional: Custom base URL (default: https://scrapebadger.com)
+  baseUrl: "https://scrapebadger.com",
 
   // Optional: Request timeout in milliseconds (default: 30000)
   timeout: 30000,
 
-  // Optional: Maximum retry attempts (default: 3)
-  maxRetries: 3,
+  // Optional: Maximum retry attempts (default: 10)
+  maxRetries: 10,
 
   // Optional: Initial retry delay in milliseconds (default: 1000)
   retryDelay: 1000,
 });
 ```
 
+### Retry Behavior
+
+The SDK automatically retries requests that fail with server errors (5xx) or rate
+limits (429) using exponential backoff (1s, 2s, 4s, 8s, ...). Each retry prints a
+colored warning:
+
+```
+⚠ ScrapeBadger: 503 Service Unavailable — retrying in 4s (attempt 3/10)
+```
+
+### Rate Limit Aware Pagination
+
+When using `*All` pagination methods (e.g. `searchAll`, `getFollowersAll`), the SDK
+reads `X-RateLimit-Remaining` and `X-RateLimit-Reset` headers from each response.
+When remaining requests drop below 20% of your tier's limit, pagination automatically
+slows down to spread requests across the remaining window — preventing 429 errors:
+
+```
+⚠ ScrapeBadger: Rate limit: 25/300 remaining (resets in 42s), throttling pagination
+```
+
+This works transparently with all tier levels (Free: 60/min, Basic: 300/min,
+Pro: 1000/min, Enterprise: 5000/min).
+```
+
 ## API Reference
 
 ### Client

package/dist/{index-DQ_jDTcQ.d.cts → index-CIZUd1Zr.d.cts}

@@ -27,6 +27,15 @@ interface ResolvedConfig {
  * Base HTTP client with retry logic and error handling.
  */
 
+interface RateLimit {
+    limit: number;
+    remaining: number;
+    reset: number;
+}
+interface ResponseWithHeaders<T> {
+    data: T;
+    rateLimit?: RateLimit;
+}
 interface RequestOptions {
     method?: "GET" | "POST" | "PUT" | "DELETE" | "PATCH";
     params?: Record<string, string | number | boolean | undefined>;
@@ -43,10 +52,22 @@ declare class BaseClient {
      * Make an HTTP request to the API.
      */
     request<T>(path: string, options?: RequestOptions): Promise<T>;
+    /**
+     * Make an HTTP request and return both data and rate limit headers.
+     */
+    requestWithHeaders<T>(path: string, options?: RequestOptions): Promise<ResponseWithHeaders<T>>;
+    /**
+     * Internal method that builds the request and executes it, returning data and rate limit info.
+     */
+    private requestRaw;
     /**
      * Execute request with exponential backoff retry logic.
      */
     private executeWithRetry;
+    /**
+     * Parse rate limit headers from an HTTP response.
+     */
+    private parseRateLimitHeaders;
     /**
      * Fetch with timeout support.
      */
@@ -64,6 +85,7 @@ declare class BaseClient {
 /**
  * Pagination utilities for the ScrapeBadger SDK.
  */
+
 /**
  * Response wrapper for paginated API responses.
  */
@@ -306,6 +328,10 @@ interface Tweet {
     in_reply_to_status_id?: string;
     /** ID of the user being replied to */
     in_reply_to_user_id?: string;
+    /** Screen name of the user being replied to */
+    in_reply_to_screen_name?: string;
+    /** Character range [start, end] of the display text within the full text */
+    display_text_range?: number[];
     /** List of attached media */
     media: Media[];
     /** List of URLs in the tweet */
@@ -372,6 +398,10 @@ interface User {
     profile_image_url?: string;
     /** Banner image URL */
     profile_banner_url?: string;
+    /** Shape of the profile image (e.g. "Circle") */
+    profile_image_shape?: string;
+    /** Interstitial type shown before viewing the profile */
+    profile_interstitial_type?: string;
     /** Number of followers */
     followers_count?: number;
     /** Number of accounts followed */
@@ -390,6 +420,8 @@ interface User {
     verified_type?: string;
     /** Whether the user has Twitter Blue */
     is_blue_verified?: boolean;
+    /** Structured verification metadata returned by the API */
+    verification_info?: Record<string, unknown>;
     /** Account creation timestamp */
     created_at?: string;
     /** Default profile */
@@ -428,6 +460,14 @@ interface User {
     professional_type?: string;
     /** Advertiser account type */
     advertiser_account_type?: string;
+    /** Number of creator subscriptions (Super Follows) this user has */
+    creator_subscriptions_count?: number;
+    /** Whether the user is eligible to offer Super Follows */
+    super_follow_eligible?: boolean;
+    /** Whether the authenticated user is Super Following this user */
+    super_following?: boolean;
+    /** Whether this user is Super Following the authenticated user */
+    super_followed_by?: boolean;
     /** IDs of pinned tweets */
     pinned_tweet_ids?: string[];
     /** Countries where withheld */
@@ -526,6 +566,18 @@ interface List {
     user_id?: string;
     /** Owner's username */
     username?: string;
+    /** Profile image URLs of a selection of list members (facepile) */
+    facepile_urls?: string[];
+    /** Human-readable followers context, e.g. "2.4K followers including @user" */
+    followers_context?: string;
+    /** Human-readable members count string, e.g. "87 members" */
+    members_context?: string;
+    /** Owner's user ID (alternative field name returned by some endpoints) */
+    owner_id?: string;
+    /** Owner's username (alternative field name returned by some endpoints) */
+    owner_username?: string;
+    /** Owner's display name */
+    owner_name?: string;
 }
 /**
  * Banner image for a community.
@@ -606,6 +658,10 @@ interface Community {
     admin_name?: string;
     /** List of community rules */
     rules?: CommunityRule[];
+    /** Primary topic for the community, e.g. "Software" */
+    primary_community_topic?: string;
+    /** Custom banner image distinct from the standard banner */
+    custom_banner?: CommunityBanner;
 }
 /**
  * A member of a community.
@@ -721,6 +777,104 @@ interface Place {
     /** Additional place attributes */
     attributes?: Record<string, string>;
 }
+/**
+ * A long-form article on Twitter.
+ */
+interface Article {
+    /** Article identifier */
+    id: string;
+    /** Article title */
+    title?: string;
+    /** Article text content */
+    text?: string;
+    /** Cover image URL */
+    cover_image_url?: string;
+    /** Creation timestamp */
+    created_at?: string;
+}
+/**
+ * A community note (Birdwatch) on a tweet.
+ */
+interface CommunityNote {
+    /** Note identifier */
+    id?: string;
+    /** Note text content */
+    text?: string;
+    /** Creation timestamp */
+    created_at?: string;
+    /** Note status */
+    status?: string;
+}
+/**
+ * A Twitter Space (audio room).
+ */
+interface Space {
+    /** Space identifier */
+    id: string;
+    /** Space title */
+    title?: string;
+    /** Current state (e.g. 'Running', 'Ended') */
+    state?: string;
+    /** Creation timestamp */
+    created_at?: string;
+    /** Start timestamp */
+    started_at?: string;
+    /** End timestamp */
+    ended_at?: string;
+    /** Scheduled start time */
+    scheduled_start?: string;
+    /** Last updated timestamp */
+    updated_at?: string;
+    /** Whether the space is locked */
+    is_locked?: boolean;
+    /** Whether replay is available */
+    is_space_available_for_replay?: boolean;
+    /** Whether clipping is available */
+    is_space_available_for_clipping?: boolean;
+    /** Total replay views */
+    total_replay_watched?: number;
+    /** Peak live listeners */
+    total_live_listeners?: number;
+    /** Media key for the space */
+    media_key?: string;
+    /** Content type (e.g. 'visual_audio') */
+    content_type?: string;
+    /** Creator's user ID */
+    creator_id?: string;
+    /** Creator's username */
+    creator_username?: string;
+    /** Creator's display name */
+    creator_name?: string;
+    /** Number of admins */
+    admin_count?: number;
+    /** Number of speakers */
+    speaker_count?: number;
+    /** Number of listeners */
+    listener_count?: number;
+}
+/**
+ * A live video broadcast.
+ */
+interface Broadcast {
+    /** Broadcast identifier */
+    id: string;
+    /** Broadcast title */
+    title?: string;
+    /** Current state */
+    state?: string;
+    /** Media key */
+    media_key?: string;
+    /** Creation timestamp */
+    created_at?: string;
+    /** Start timestamp */
+    started_at?: string;
+    /** Video width */
+    width?: number;
+    /** Video height */
+    height?: number;
+    /** Total viewer count */
+    total_viewers?: number;
+}
 /**
  * Generic API response with data and optional cursor.
  */
@@ -1015,6 +1169,47 @@ declare class TweetsClient {
      * ```
      */
     getUserTweetsAll(username: string, options?: IteratorOptions): AsyncGenerator<Tweet, void, undefined>;
+    /**
+     * Get the edit history of a tweet.
+     *
+     * @param tweetId - The tweet ID to get edit history for.
+     * @returns Paginated response containing tweet versions.
+     *
+     * @example
+     * ```typescript
+     * const history = await client.twitter.tweets.getEditHistory("1234567890");
+     * console.log(`${history.data.length} version(s) of this tweet`);
+     * ```
+     */
+    getEditHistory(tweetId: string): Promise<PaginatedResponse<Tweet>>;
+    /**
+     * Get community notes (Birdwatch) attached to a tweet.
+     *
+     * @param tweetId - The tweet ID to get community notes for.
+     * @returns Paginated response containing community notes.
+     *
+     * @example
+     * ```typescript
+     * const notes = await client.twitter.tweets.getCommunityNotes("1234567890");
+     * for (const note of notes.data) {
+     *   console.log(note.text);
+     * }
+     * ```
+     */
+    getCommunityNotes(tweetId: string): Promise<PaginatedResponse<CommunityNote>>;
+    /**
+     * Get a long-form article by its ID.
+     *
+     * @param articleId - The article ID to fetch.
+     * @returns The article data.
+     *
+     * @example
+     * ```typescript
+     * const article = await client.twitter.tweets.getArticle("abc123");
+     * console.log(`${article.title}: ${article.text?.slice(0, 100)}...`);
+     * ```
+     */
+    getArticle(articleId: string): Promise<Article>;
 }
 
 /**
@@ -1272,6 +1467,72 @@ declare class UsersClient {
      * @yields User objects matching the search query.
      */
     searchAll(query: string, options?: IteratorOptions): AsyncGenerator<User, void, undefined>;
+    /**
+     * Get multiple users by their numeric IDs in a single request.
+     *
+     * @param userIds - List of user IDs to fetch.
+     * @returns Paginated response containing the matching users.
+     *
+     * @example
+     * ```typescript
+     * const users = await client.twitter.users.getByIds(["44196397", "783214"]);
+     * for (const user of users.data) {
+     *   console.log(`@${user.username}`);
+     * }
+     * ```
+     */
+    getByIds(userIds: string[]): Promise<PaginatedResponse<User>>;
+    /**
+     * Get multiple users by their usernames in a single request.
+     *
+     * @param usernames - List of usernames (without @) to fetch.
+     * @returns Paginated response containing the matching users.
+     *
+     * @example
+     * ```typescript
+     * const users = await client.twitter.users.getByUsernames(["elonmusk", "twitter"]);
+     * for (const user of users.data) {
+     *   console.log(`${user.name}: ${user.followers_count?.toLocaleString()} followers`);
+     * }
+     * ```
+     */
+    getByUsernames(usernames: string[]): Promise<PaginatedResponse<User>>;
+    /**
+     * Get tweets that mention a user.
+     *
+     * @param username - The user's username (without @).
+     * @param options - Pagination options with optional count.
+     * @returns Paginated response containing tweets mentioning the user.
+     *
+     * @example
+     * ```typescript
+     * const mentions = await client.twitter.users.getMentions("elonmusk");
+     * for (const tweet of mentions.data) {
+     *   console.log(`@${tweet.username}: ${tweet.text.slice(0, 100)}...`);
+     * }
+     * ```
+     */
+    getMentions(username: string, options?: PaginationOptions & {
+        count?: number;
+    }): Promise<PaginatedResponse<Tweet>>;
+    /**
+     * Get long-form articles authored by a user.
+     *
+     * @param userId - The user's numeric ID.
+     * @param options - Pagination options with optional count.
+     * @returns Paginated response containing the user's articles as tweets.
+     *
+     * @example
+     * ```typescript
+     * const articles = await client.twitter.users.getArticles("44196397");
+     * for (const article of articles.data) {
+     *   console.log(article.text?.slice(0, 100));
+     * }
+     * ```
+     */
+    getArticles(userId: string, options?: PaginationOptions & {
+        count?: number;
+    }): Promise<PaginatedResponse<Tweet>>;
 }
 
 /**
@@ -1404,6 +1665,25 @@ declare class ListsClient {
     getMyLists(options?: PaginationOptions & {
         count?: number;
     }): Promise<PaginatedResponse<List>>;
+    /**
+     * Search tweets within a specific list.
+     *
+     * @param listId - The list ID to search within.
+     * @param query - Search query string.
+     * @param options - Pagination options with optional count.
+     * @returns Paginated response containing matching tweets from the list.
+     *
+     * @example
+     * ```typescript
+     * const results = await client.twitter.lists.searchTweets("123456", "python");
+     * for (const tweet of results.data) {
+     *   console.log(`@${tweet.username}: ${tweet.text.slice(0, 100)}...`);
+     * }
+     * ```
+     */
+    searchTweets(listId: string, query: string, options?: PaginationOptions & {
+        count?: number;
+    }): Promise<PaginatedResponse<Tweet>>;
 }
 
 /**
@@ -2707,6 +2987,63 @@ declare class StreamClient {
  */
 declare function verifyWebhookSignature(secret: string, body: string | Buffer, signatureHeader: string): boolean;
 
+/**
+ * Twitter Spaces API client.
+ *
+ * Provides methods for fetching Twitter Spaces and live broadcasts.
+ */
+
+/**
+ * Client for Twitter Spaces endpoints.
+ *
+ * Provides async methods for fetching Space details and live broadcast data.
+ *
+ * @example
+ * ```typescript
+ * const client = new ScrapeBadger({ apiKey: "key" });
+ *
+ * // Get Space details
+ * const space = await client.twitter.spaces.getDetail("1eaKbrPPbPwKX");
+ * console.log(`${space.title} — state: ${space.state}`);
+ *
+ * // Get broadcast details
+ * const broadcast = await client.twitter.spaces.getBroadcast("broadcast123");
+ * console.log(`${broadcast.title}: ${broadcast.total_viewers} viewers`);
+ * ```
+ */
+declare class SpacesClient {
+    private readonly client;
+    constructor(client: BaseClient);
+    /**
+     * Get details for a specific Twitter Space.
+     *
+     * @param spaceId - The Space ID to fetch.
+     * @returns The Space data.
+     * @throws NotFoundError - If the Space doesn't exist.
+     *
+     * @example
+     * ```typescript
+     * const space = await client.twitter.spaces.getDetail("1eaKbrPPbPwKX");
+     * console.log(`${space.title} — ${space.participant_count} participants`);
+     * ```
+     */
+    getDetail(spaceId: string): Promise<Space>;
+    /**
+     * Get details for a live video broadcast.
+     *
+     * @param broadcastId - The broadcast ID to fetch.
+     * @returns The broadcast data.
+     * @throws NotFoundError - If the broadcast doesn't exist.
+     *
+     * @example
+     * ```typescript
+     * const broadcast = await client.twitter.spaces.getBroadcast("broadcast123");
+     * console.log(`${broadcast.title}: ${broadcast.total_viewers} viewers`);
+     * ```
+     */
+    getBroadcast(broadcastId: string): Promise<Broadcast>;
+}
+
 /**
  * Twitter API client.
  *
@@ -2724,6 +3061,7 @@ declare function verifyWebhookSignature(secret: string, body: string | Buffer, s
  * - `trends` - Trending topics and locations
  * - `geo` - Geographic place information
  * - `stream` - Real-time stream monitor management and WebSocket streaming
+ * - `spaces` - Twitter Spaces and live broadcast details
  *
  * @example
  * ```typescript
@@ -2763,6 +3101,8 @@ declare class TwitterClient {
     readonly geo: GeoClient;
     /** Client for real-time stream monitor management and WebSocket streaming */
     readonly stream: StreamClient;
+    /** Client for Twitter Spaces and live broadcast operations */
+    readonly spaces: SpacesClient;
     /**
      * Create a new Twitter client.
      *
@@ -2771,4 +3111,4 @@ declare class TwitterClient {
     constructor(client: BaseClient);
 }
 
-export { type ListResponse as $, AuthenticationError as A, BaseClient as B, ConflictError as C, type List as D, type CommunityBanner as E, type CommunityRule as F, GeoClient as G, type Hashtag as H, InsufficientCreditsError as I, type Community as J, type CommunityMember as K, ListsClient as L, type Media as M, NotFoundError as N, type Trend as O, type PaginatedResponse as P, type QueryType as Q, type ResolvedConfig as R, type ScrapeBadgerConfig as S, TwitterClient as T, UsersClient as U, ValidationError as V, WebSocketStreamError as W, type Location as X, type PlaceTrends as Y, type Place as Z, type ApiResponse as _, ScrapeBadgerError as a, type MonitorStatus as a0, type StreamMonitor as a1, type StreamMonitorList as a2, type CreateMonitorParams as a3, type UpdateMonitorParams as a4, type StreamTweet as a5, type ConnectedEvent as a6, type PingEvent as a7, type TweetEvent as a8, type ErrorEvent as a9, type StreamEvent as aa, type StreamEventType as ab, type DeliveryLog as ac, type DeliveryLogList as ad, type BillingLog as ae, type BillingLogList as af, type ConnectOptions as ag, type FilterRuleStatus as ah, type FilterRuleResponse as ai, type FilterRuleCreate as aj, type FilterRuleUpdate as ak, type FilterRulePricingTier as al, type FilterRuleListResponse as am, type FilterRuleValidateResponse as an, type FilterRuleDeliveryLog as ao, type FilterRuleDeliveryLogListResponse as ap, type FilterRulePricingTiersResponse as aq, RateLimitError as b, ServerError as c, TimeoutError as d, AccountRestrictedError as e, type PaginationOptions as f, type IteratorOptions as g, collectAll as h, TweetsClient as i, CommunitiesClient as j, TrendsClient as k, type GeoSearchOptions as l, StreamClient as m, type StreamEmitter as n, type TrendCategory as o, type CommunityTweetType as p, type PollOption as q, type Poll as r, type Url as s, type UserMention as t, type TweetPlace as u, verifyWebhookSignature as v, type Tweet as w, type User as x, type UserAbout as y, type UserIds as z };
+export { type Article as $, AuthenticationError as A, BaseClient as B, ConflictError as C, type UserIds as D, type List as E, type CommunityBanner as F, GeoClient as G, type Hashtag as H, InsufficientCreditsError as I, type CommunityRule as J, type Community as K, ListsClient as L, type Media as M, NotFoundError as N, type CommunityMember as O, type PaginatedResponse as P, type QueryType as Q, type ResolvedConfig as R, type ScrapeBadgerConfig as S, TwitterClient as T, UsersClient as U, ValidationError as V, WebSocketStreamError as W, type Trend as X, type Location as Y, type PlaceTrends as Z, type Place as _, ScrapeBadgerError as a, type CommunityNote as a0, type Space as a1, type Broadcast as a2, type ApiResponse as a3, type ListResponse as a4, type MonitorStatus as a5, type StreamMonitor as a6, type StreamMonitorList as a7, type CreateMonitorParams as a8, type UpdateMonitorParams as a9, type StreamTweet as aa, type ConnectedEvent as ab, type PingEvent as ac, type TweetEvent as ad, type ErrorEvent as ae, type StreamEvent as af, type StreamEventType as ag, type DeliveryLog as ah, type DeliveryLogList as ai, type BillingLog as aj, type BillingLogList as ak, type ConnectOptions as al, type FilterRuleStatus as am, type FilterRuleResponse as an, type FilterRuleCreate as ao, type FilterRuleUpdate as ap, type FilterRulePricingTier as aq, type FilterRuleListResponse as ar, type FilterRuleValidateResponse as as, type FilterRuleDeliveryLog as at, type FilterRuleDeliveryLogListResponse as au, type FilterRulePricingTiersResponse as av, RateLimitError as b, ServerError as c, TimeoutError as d, AccountRestrictedError as e, type PaginationOptions as f, type IteratorOptions as g, collectAll as h, TweetsClient as i, CommunitiesClient as j, TrendsClient as k, type GeoSearchOptions as l, StreamClient as m, type StreamEmitter as n, SpacesClient as o, type TrendCategory as p, type CommunityTweetType as q, type PollOption as r, type Poll as s, type Url as t, type UserMention as u, verifyWebhookSignature as v, type TweetPlace as w, type Tweet as x, type User as y, type UserAbout as z };