scrapebadger 0.3.0 → 0.4.0

package/dist/index.js

@@ -130,6 +130,19 @@ var BaseClient = class {
    * Make an HTTP request to the API.
    */
   async request(path, options = {}) {
+    const { data } = await this.requestRaw(path, options);
+    return data;
+  }
+  /**
+   * Make an HTTP request and return both data and rate limit headers.
+   */
+  async requestWithHeaders(path, options = {}) {
+    return this.requestRaw(path, options);
+  }
+  /**
+   * Internal method that builds the request and executes it, returning data and rate limit info.
+   */
+  async requestRaw(path, options = {}) {
     const { method = "GET", params, body, headers = {} } = options;
     const url = new URL(path, this.config.baseUrl);
     if (params) {
@@ -143,7 +156,7 @@ var BaseClient = class {
       "Content-Type": "application/json",
       Accept: "application/json",
       "X-API-Key": this.config.apiKey,
-      "User-Agent": "scrapebadger-node/0.1.0",
+      "User-Agent": "scrapebadger-node/0.3.1",
       ...headers
     };
     const fetchOptions = {
@@ -162,8 +175,10 @@ var BaseClient = class {
     let lastError;
     for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) {
       try {
-        const response = await this.fetchWithTimeout(url, options);
-        return await this.handleResponse(response);
+        const httpResponse = await this.fetchWithTimeout(url, options);
+        const data = await this.handleResponse(httpResponse);
+        const rateLimit = this.parseRateLimitHeaders(httpResponse.headers);
+        return { data, rateLimit };
       } catch (error) {
         lastError = error;
         if (error instanceof ScrapeBadgerError && !(error instanceof RateLimitError)) {
@@ -173,18 +188,56 @@ var BaseClient = class {
           break;
         }
         const delay = this.config.retryDelay * Math.pow(2, attempt);
-        if (error instanceof RateLimitError && error.retryAfter) {
-          const retryDelay = (error.retryAfter - Date.now() / 1e3) * 1e3;
-          if (retryDelay > 0 && retryDelay < 6e4) {
-            await this.sleep(retryDelay);
-            continue;
+        const delaySec = Math.round(delay / 1e3);
+        const attemptNum = attempt + 1;
+        const maxRetries = this.config.maxRetries;
+        if (error instanceof RateLimitError) {
+          console.warn(
+            `\x1B[33m\u26A0 ScrapeBadger: 429 Rate Limited \u2014 retrying in ${delaySec}s (attempt ${attemptNum}/${maxRetries})\x1B[0m`
+          );
+          if (error.retryAfter) {
+            const retryDelay = (error.retryAfter - Date.now() / 1e3) * 1e3;
+            if (retryDelay > 0 && retryDelay < 6e4) {
+              await this.sleep(retryDelay);
+              continue;
+            }
           }
+        } else if (error instanceof TimeoutError) {
+          console.warn(
+            `\x1B[33m\u26A0 ScrapeBadger: TimeoutError \u2014 retrying in ${delaySec}s (attempt ${attemptNum}/${maxRetries})\x1B[0m`
+          );
+        } else if (error instanceof ServerError) {
+          console.warn(
+            `\x1B[33m\u26A0 ScrapeBadger: ${error.statusCode} ${error.message} \u2014 retrying in ${delaySec}s (attempt ${attemptNum}/${maxRetries})\x1B[0m`
+          );
+        } else {
+          console.warn(
+            `\x1B[33m\u26A0 ScrapeBadger: ${error.name} \u2014 retrying in ${delaySec}s (attempt ${attemptNum}/${maxRetries})\x1B[0m`
+          );
         }
         await this.sleep(delay);
       }
     }
     throw lastError ?? new ScrapeBadgerError("Request failed after retries");
   }
+  /**
+   * Parse rate limit headers from an HTTP response.
+   */
+  parseRateLimitHeaders(headers) {
+    const limit = headers.get("X-RateLimit-Limit");
+    const remaining = headers.get("X-RateLimit-Remaining");
+    const reset = headers.get("X-RateLimit-Reset");
+    if (limit === null || remaining === null || reset === null) {
+      return void 0;
+    }
+    const parsedLimit = parseInt(limit, 10);
+    const parsedRemaining = parseInt(remaining, 10);
+    const parsedReset = parseInt(reset, 10);
+    if (isNaN(parsedLimit) || isNaN(parsedRemaining) || isNaN(parsedReset)) {
+      return void 0;
+    }
+    return { limit: parsedLimit, remaining: parsedRemaining, reset: parsedReset };
+  }
   /**
    * Fetch with timeout support.
    */
@@ -266,7 +319,7 @@ var BaseClient = class {
 // src/internal/config.ts
 var DEFAULT_BASE_URL = "https://scrapebadger.com";
 var DEFAULT_TIMEOUT = 3e4;
-var DEFAULT_MAX_RETRIES = 3;
+var DEFAULT_MAX_RETRIES = 10;
 var DEFAULT_RETRY_DELAY = 1e3;
 function resolveConfig(config) {
   if (!config.apiKey) {
@@ -295,12 +348,26 @@ function createPaginatedResponse(data, cursor) {
     hasMore: !!cursor
   };
 }
+var RATE_LIMIT_WARN_THRESHOLD = 0.2;
 async function* paginate(fetchPage, options = {}) {
   const { maxItems } = options;
   let cursor;
   let totalYielded = 0;
   do {
-    const response = await fetchPage(cursor);
+    const { response, rateLimit } = await fetchPage(cursor);
+    if (rateLimit) {
+      const { limit, remaining, reset } = rateLimit;
+      if (limit > 0 && remaining / limit < RATE_LIMIT_WARN_THRESHOLD) {
+        const nowSec = Date.now() / 1e3;
+        const windowRemainingSec = Math.max(reset - nowSec, 1);
+        const delayMs = remaining > 0 ? windowRemainingSec / remaining * 1e3 : windowRemainingSec * 1e3;
+        const resetInSec = Math.round(windowRemainingSec);
+        console.warn(
+          `\x1B[33m\u26A0 ScrapeBadger: Rate limit: ${remaining}/${limit} remaining (resets in ${resetInSec}s), throttling pagination\x1B[0m`
+        );
+        await sleep(delayMs);
+      }
+    }
     for (const item of response.data) {
       yield item;
       totalYielded++;
@@ -318,6 +385,9 @@ async function collectAll(generator) {
   }
   return items;
 }
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 
 // src/twitter/tweets.ts
 var TweetsClient = class {
@@ -515,7 +585,8 @@ var TweetsClient = class {
    */
   async *getQuotesAll(tweetId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getQuotes(tweetId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/tweets/tweet/${tweetId}/quotes`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -582,7 +653,15 @@ var TweetsClient = class {
    */
   async *searchAll(query, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.search(query, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders("/v1/twitter/tweets/advanced_search", {
+        params: {
+          query,
+          query_type: options.queryType ?? "Top",
+          count: options.count,
+          cursor
+        }
+      });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -626,10 +705,64 @@ var TweetsClient = class {
    */
   async *getUserTweetsAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getUserTweets(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/latest_tweets`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
+  /**
+   * Get the edit history of a tweet.
+   *
+   * @param tweetId - The tweet ID to get edit history for.
+   * @returns Paginated response containing tweet versions.
+   *
+   * @example
+   * ```typescript
+   * const history = await client.twitter.tweets.getEditHistory("1234567890");
+   * console.log(`${history.data.length} version(s) of this tweet`);
+   * ```
+   */
+  async getEditHistory(tweetId) {
+    const response = await this.client.request(
+      `/v1/twitter/tweets/tweet/${tweetId}/edit_history`
+    );
+    return createPaginatedResponse(response.data ?? [], void 0);
+  }
+  /**
+   * Get community notes (Birdwatch) attached to a tweet.
+   *
+   * @param tweetId - The tweet ID to get community notes for.
+   * @returns Paginated response containing community notes.
+   *
+   * @example
+   * ```typescript
+   * const notes = await client.twitter.tweets.getCommunityNotes("1234567890");
+   * for (const note of notes.data) {
+   *   console.log(note.text);
+   * }
+   * ```
+   */
+  async getCommunityNotes(tweetId) {
+    const response = await this.client.request(
+      `/v1/twitter/tweets/tweet/${tweetId}/community_notes`
+    );
+    return createPaginatedResponse(response.data ?? [], void 0);
+  }
+  /**
+   * Get a long-form article by its ID.
+   *
+   * @param articleId - The article ID to fetch.
+   * @returns The article data.
+   *
+   * @example
+   * ```typescript
+   * const article = await client.twitter.tweets.getArticle("abc123");
+   * console.log(`${article.title}: ${article.text?.slice(0, 100)}...`);
+   * ```
+   */
+  async getArticle(articleId) {
+    return this.client.request(`/v1/twitter/tweets/article/${articleId}`);
+  }
 };
 
 // src/twitter/users.ts
@@ -736,7 +869,8 @@ var UsersClient = class {
    */
   async *getFollowersAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getFollowers(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/followers`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -771,7 +905,8 @@ var UsersClient = class {
    */
   async *getFollowingAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getFollowing(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/followings`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -934,10 +1069,97 @@ var UsersClient = class {
    */
   async *searchAll(query, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.search(query, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders("/v1/twitter/users/search_users", { params: { query, cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
+  /**
+   * Get multiple users by their numeric IDs in a single request.
+   *
+   * @param userIds - List of user IDs to fetch.
+   * @returns Paginated response containing the matching users.
+   *
+   * @example
+   * ```typescript
+   * const users = await client.twitter.users.getByIds(["44196397", "783214"]);
+   * for (const user of users.data) {
+   *   console.log(`@${user.username}`);
+   * }
+   * ```
+   */
+  async getByIds(userIds) {
+    const response = await this.client.request(
+      "/v1/twitter/users/batch_by_ids",
+      { params: { user_ids: userIds.join(",") } }
+    );
+    return createPaginatedResponse(response.data ?? [], void 0);
+  }
+  /**
+   * Get multiple users by their usernames in a single request.
+   *
+   * @param usernames - List of usernames (without @) to fetch.
+   * @returns Paginated response containing the matching users.
+   *
+   * @example
+   * ```typescript
+   * const users = await client.twitter.users.getByUsernames(["elonmusk", "twitter"]);
+   * for (const user of users.data) {
+   *   console.log(`${user.name}: ${user.followers_count?.toLocaleString()} followers`);
+   * }
+   * ```
+   */
+  async getByUsernames(usernames) {
+    const response = await this.client.request(
+      "/v1/twitter/users/batch_by_usernames",
+      { params: { usernames: usernames.join(",") } }
+    );
+    return createPaginatedResponse(response.data ?? [], void 0);
+  }
+  /**
+   * Get tweets that mention a user.
+   *
+   * @param username - The user's username (without @).
+   * @param options - Pagination options with optional count.
+   * @returns Paginated response containing tweets mentioning the user.
+   *
+   * @example
+   * ```typescript
+   * const mentions = await client.twitter.users.getMentions("elonmusk");
+   * for (const tweet of mentions.data) {
+   *   console.log(`@${tweet.username}: ${tweet.text.slice(0, 100)}...`);
+   * }
+   * ```
+   */
+  async getMentions(username, options = {}) {
+    const response = await this.client.request(
+      `/v1/twitter/users/${username}/mentions`,
+      { params: { count: options.count, cursor: options.cursor } }
+    );
+    return createPaginatedResponse(response.data ?? [], response.next_cursor);
+  }
+  /**
+   * Get long-form articles authored by a user.
+   *
+   * @param userId - The user's numeric ID.
+   * @param options - Pagination options with optional count.
+   * @returns Paginated response containing the user's articles as tweets.
+   *
+   * @example
+   * ```typescript
+   * const articles = await client.twitter.users.getArticles("44196397");
+   * for (const article of articles.data) {
+   *   console.log(article.text?.slice(0, 100));
+   * }
+   * ```
+   */
+  async getArticles(userId, options = {}) {
+    const response = await this.client.request(
+      `/v1/twitter/users/${userId}/articles`,
+      { params: { count: options.count, cursor: options.cursor } }
+    );
+    return createPaginatedResponse(response.data ?? [], response.next_cursor);
+  }
 };
 
 // src/twitter/lists.ts
@@ -994,7 +1216,8 @@ var ListsClient = class {
    */
   async *getTweetsAll(listId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getTweets(listId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/lists/${listId}/tweets`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -1029,7 +1252,8 @@ var ListsClient = class {
    */
   async *getMembersAll(listId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getMembers(listId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/lists/${listId}/members`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -1082,6 +1306,29 @@ var ListsClient = class {
     );
     return createPaginatedResponse(response.data ?? [], response.next_cursor);
   }
+  /**
+   * Search tweets within a specific list.
+   *
+   * @param listId - The list ID to search within.
+   * @param query - Search query string.
+   * @param options - Pagination options with optional count.
+   * @returns Paginated response containing matching tweets from the list.
+   *
+   * @example
+   * ```typescript
+   * const results = await client.twitter.lists.searchTweets("123456", "python");
+   * for (const tweet of results.data) {
+   *   console.log(`@${tweet.username}: ${tweet.text.slice(0, 100)}...`);
+   * }
+   * ```
+   */
+  async searchTweets(listId, query, options = {}) {
+    const response = await this.client.request(
+      `/v1/twitter/lists/${listId}/search_tweets`,
+      { params: { query, count: options.count, cursor: options.cursor } }
+    );
+    return createPaginatedResponse(response.data ?? [], response.next_cursor);
+  }
 };
 
 // src/twitter/communities.ts
@@ -1177,7 +1424,14 @@ var CommunitiesClient = class {
    */
   async *getTweetsAll(communityId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getTweets(communityId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/communities/${communityId}/tweets`, {
+        params: {
+          tweet_type: options.tweetType ?? "Top",
+          count: options.count ?? 40,
+          cursor
+        }
+      });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -2096,6 +2350,46 @@ function verifyWebhookSignature(secret, body, signatureHeader) {
   }
 }
 
+// src/twitter/spaces.ts
+var SpacesClient = class {
+  client;
+  constructor(client) {
+    this.client = client;
+  }
+  /**
+   * Get details for a specific Twitter Space.
+   *
+   * @param spaceId - The Space ID to fetch.
+   * @returns The Space data.
+   * @throws NotFoundError - If the Space doesn't exist.
+   *
+   * @example
+   * ```typescript
+   * const space = await client.twitter.spaces.getDetail("1eaKbrPPbPwKX");
+   * console.log(`${space.title} — ${space.participant_count} participants`);
+   * ```
+   */
+  async getDetail(spaceId) {
+    return this.client.request(`/v1/twitter/spaces/${spaceId}`);
+  }
+  /**
+   * Get details for a live video broadcast.
+   *
+   * @param broadcastId - The broadcast ID to fetch.
+   * @returns The broadcast data.
+   * @throws NotFoundError - If the broadcast doesn't exist.
+   *
+   * @example
+   * ```typescript
+   * const broadcast = await client.twitter.spaces.getBroadcast("broadcast123");
+   * console.log(`${broadcast.title}: ${broadcast.total_viewers} viewers`);
+   * ```
+   */
+  async getBroadcast(broadcastId) {
+    return this.client.request(`/v1/twitter/spaces/broadcast/${broadcastId}`);
+  }
+};
+
 // src/twitter/client.ts
 var TwitterClient = class {
   /** Client for tweet operations */
@@ -2112,6 +2406,8 @@ var TwitterClient = class {
   geo;
   /** Client for real-time stream monitor management and WebSocket streaming */
   stream;
+  /** Client for Twitter Spaces and live broadcast operations */
+  spaces;
   /**
    * Create a new Twitter client.
    *
@@ -2125,6 +2421,7 @@ var TwitterClient = class {
     this.trends = new TrendsClient(client);
     this.geo = new GeoClient(client);
     this.stream = new StreamClient(client);
+    this.spaces = new SpacesClient(client);
   }
 };
 
@@ -2285,6 +2582,7 @@ exports.RateLimitError = RateLimitError;
 exports.ScrapeBadger = ScrapeBadger;
 exports.ScrapeBadgerError = ScrapeBadgerError;
 exports.ServerError = ServerError;
+exports.SpacesClient = SpacesClient;
 exports.StreamClient = StreamClient;
 exports.TimeoutError = TimeoutError;
 exports.TrendsClient = TrendsClient;