scrapebadger 0.3.0 → 0.3.1

package/dist/index.mjs

@@ -124,6 +124,19 @@ var BaseClient = class {
    * Make an HTTP request to the API.
    */
   async request(path, options = {}) {
+    const { data } = await this.requestRaw(path, options);
+    return data;
+  }
+  /**
+   * Make an HTTP request and return both data and rate limit headers.
+   */
+  async requestWithHeaders(path, options = {}) {
+    return this.requestRaw(path, options);
+  }
+  /**
+   * Internal method that builds the request and executes it, returning data and rate limit info.
+   */
+  async requestRaw(path, options = {}) {
     const { method = "GET", params, body, headers = {} } = options;
     const url = new URL(path, this.config.baseUrl);
     if (params) {
@@ -137,7 +150,7 @@ var BaseClient = class {
       "Content-Type": "application/json",
       Accept: "application/json",
       "X-API-Key": this.config.apiKey,
-      "User-Agent": "scrapebadger-node/0.1.0",
+      "User-Agent": "scrapebadger-node/0.3.1",
       ...headers
     };
     const fetchOptions = {
@@ -156,8 +169,10 @@ var BaseClient = class {
     let lastError;
     for (let attempt = 0; attempt <= this.config.maxRetries; attempt++) {
       try {
-        const response = await this.fetchWithTimeout(url, options);
-        return await this.handleResponse(response);
+        const httpResponse = await this.fetchWithTimeout(url, options);
+        const data = await this.handleResponse(httpResponse);
+        const rateLimit = this.parseRateLimitHeaders(httpResponse.headers);
+        return { data, rateLimit };
       } catch (error) {
         lastError = error;
         if (error instanceof ScrapeBadgerError && !(error instanceof RateLimitError)) {
@@ -167,18 +182,56 @@ var BaseClient = class {
           break;
         }
         const delay = this.config.retryDelay * Math.pow(2, attempt);
-        if (error instanceof RateLimitError && error.retryAfter) {
-          const retryDelay = (error.retryAfter - Date.now() / 1e3) * 1e3;
-          if (retryDelay > 0 && retryDelay < 6e4) {
-            await this.sleep(retryDelay);
-            continue;
+        const delaySec = Math.round(delay / 1e3);
+        const attemptNum = attempt + 1;
+        const maxRetries = this.config.maxRetries;
+        if (error instanceof RateLimitError) {
+          console.warn(
+            `\x1B[33m\u26A0 ScrapeBadger: 429 Rate Limited \u2014 retrying in ${delaySec}s (attempt ${attemptNum}/${maxRetries})\x1B[0m`
+          );
+          if (error.retryAfter) {
+            const retryDelay = (error.retryAfter - Date.now() / 1e3) * 1e3;
+            if (retryDelay > 0 && retryDelay < 6e4) {
+              await this.sleep(retryDelay);
+              continue;
+            }
           }
+        } else if (error instanceof TimeoutError) {
+          console.warn(
+            `\x1B[33m\u26A0 ScrapeBadger: TimeoutError \u2014 retrying in ${delaySec}s (attempt ${attemptNum}/${maxRetries})\x1B[0m`
+          );
+        } else if (error instanceof ServerError) {
+          console.warn(
+            `\x1B[33m\u26A0 ScrapeBadger: ${error.statusCode} ${error.message} \u2014 retrying in ${delaySec}s (attempt ${attemptNum}/${maxRetries})\x1B[0m`
+          );
+        } else {
+          console.warn(
+            `\x1B[33m\u26A0 ScrapeBadger: ${error.name} \u2014 retrying in ${delaySec}s (attempt ${attemptNum}/${maxRetries})\x1B[0m`
+          );
         }
         await this.sleep(delay);
       }
     }
     throw lastError ?? new ScrapeBadgerError("Request failed after retries");
   }
+  /**
+   * Parse rate limit headers from an HTTP response.
+   */
+  parseRateLimitHeaders(headers) {
+    const limit = headers.get("X-RateLimit-Limit");
+    const remaining = headers.get("X-RateLimit-Remaining");
+    const reset = headers.get("X-RateLimit-Reset");
+    if (limit === null || remaining === null || reset === null) {
+      return void 0;
+    }
+    const parsedLimit = parseInt(limit, 10);
+    const parsedRemaining = parseInt(remaining, 10);
+    const parsedReset = parseInt(reset, 10);
+    if (isNaN(parsedLimit) || isNaN(parsedRemaining) || isNaN(parsedReset)) {
+      return void 0;
+    }
+    return { limit: parsedLimit, remaining: parsedRemaining, reset: parsedReset };
+  }
   /**
    * Fetch with timeout support.
    */
@@ -260,7 +313,7 @@ var BaseClient = class {
 // src/internal/config.ts
 var DEFAULT_BASE_URL = "https://scrapebadger.com";
 var DEFAULT_TIMEOUT = 3e4;
-var DEFAULT_MAX_RETRIES = 3;
+var DEFAULT_MAX_RETRIES = 10;
 var DEFAULT_RETRY_DELAY = 1e3;
 function resolveConfig(config) {
   if (!config.apiKey) {
@@ -289,12 +342,26 @@ function createPaginatedResponse(data, cursor) {
     hasMore: !!cursor
   };
 }
+var RATE_LIMIT_WARN_THRESHOLD = 0.2;
 async function* paginate(fetchPage, options = {}) {
   const { maxItems } = options;
   let cursor;
   let totalYielded = 0;
   do {
-    const response = await fetchPage(cursor);
+    const { response, rateLimit } = await fetchPage(cursor);
+    if (rateLimit) {
+      const { limit, remaining, reset } = rateLimit;
+      if (limit > 0 && remaining / limit < RATE_LIMIT_WARN_THRESHOLD) {
+        const nowSec = Date.now() / 1e3;
+        const windowRemainingSec = Math.max(reset - nowSec, 1);
+        const delayMs = remaining > 0 ? windowRemainingSec / remaining * 1e3 : windowRemainingSec * 1e3;
+        const resetInSec = Math.round(windowRemainingSec);
+        console.warn(
+          `\x1B[33m\u26A0 ScrapeBadger: Rate limit: ${remaining}/${limit} remaining (resets in ${resetInSec}s), throttling pagination\x1B[0m`
+        );
+        await sleep(delayMs);
+      }
+    }
     for (const item of response.data) {
       yield item;
       totalYielded++;
@@ -312,6 +379,9 @@ async function collectAll(generator) {
   }
   return items;
 }
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 
 // src/twitter/tweets.ts
 var TweetsClient = class {
@@ -509,7 +579,8 @@ var TweetsClient = class {
    */
   async *getQuotesAll(tweetId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getQuotes(tweetId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/tweets/tweet/${tweetId}/quotes`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -576,7 +647,15 @@ var TweetsClient = class {
    */
   async *searchAll(query, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.search(query, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders("/v1/twitter/tweets/advanced_search", {
+        params: {
+          query,
+          query_type: options.queryType ?? "Top",
+          count: options.count,
+          cursor
+        }
+      });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -620,7 +699,8 @@ var TweetsClient = class {
    */
   async *getUserTweetsAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getUserTweets(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/latest_tweets`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -730,7 +810,8 @@ var UsersClient = class {
    */
   async *getFollowersAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getFollowers(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/followers`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -765,7 +846,8 @@ var UsersClient = class {
    */
   async *getFollowingAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getFollowing(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/followings`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -928,7 +1010,8 @@ var UsersClient = class {
    */
   async *searchAll(query, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.search(query, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders("/v1/twitter/users/search_users", { params: { query, cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -988,7 +1071,8 @@ var ListsClient = class {
    */
   async *getTweetsAll(listId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getTweets(listId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/lists/${listId}/tweets`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -1023,7 +1107,8 @@ var ListsClient = class {
    */
   async *getMembersAll(listId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getMembers(listId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/lists/${listId}/members`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -1171,7 +1256,14 @@ var CommunitiesClient = class {
    */
   async *getTweetsAll(communityId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getTweets(communityId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/communities/${communityId}/tweets`, {
+        params: {
+          tweet_type: options.tweetType ?? "Top",
+          count: options.count ?? 40,
+          cursor
+        }
+      });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }