scrapebadger 0.3.0 → 0.3.1

package/dist/twitter/index.mjs

@@ -10,12 +10,26 @@ function createPaginatedResponse(data, cursor) {
     hasMore: !!cursor
   };
 }
+var RATE_LIMIT_WARN_THRESHOLD = 0.2;
 async function* paginate(fetchPage, options = {}) {
   const { maxItems } = options;
   let cursor;
   let totalYielded = 0;
   do {
-    const response = await fetchPage(cursor);
+    const { response, rateLimit } = await fetchPage(cursor);
+    if (rateLimit) {
+      const { limit, remaining, reset } = rateLimit;
+      if (limit > 0 && remaining / limit < RATE_LIMIT_WARN_THRESHOLD) {
+        const nowSec = Date.now() / 1e3;
+        const windowRemainingSec = Math.max(reset - nowSec, 1);
+        const delayMs = remaining > 0 ? windowRemainingSec / remaining * 1e3 : windowRemainingSec * 1e3;
+        const resetInSec = Math.round(windowRemainingSec);
+        console.warn(
+          `\x1B[33m\u26A0 ScrapeBadger: Rate limit: ${remaining}/${limit} remaining (resets in ${resetInSec}s), throttling pagination\x1B[0m`
+        );
+        await sleep(delayMs);
+      }
+    }
     for (const item of response.data) {
       yield item;
       totalYielded++;
@@ -26,6 +40,9 @@ async function* paginate(fetchPage, options = {}) {
     cursor = response.nextCursor;
   } while (cursor);
 }
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 
 // src/twitter/tweets.ts
 var TweetsClient = class {
@@ -223,7 +240,8 @@ var TweetsClient = class {
    */
   async *getQuotesAll(tweetId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getQuotes(tweetId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/tweets/tweet/${tweetId}/quotes`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -290,7 +308,15 @@ var TweetsClient = class {
    */
   async *searchAll(query, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.search(query, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders("/v1/twitter/tweets/advanced_search", {
+        params: {
+          query,
+          query_type: options.queryType ?? "Top",
+          count: options.count,
+          cursor
+        }
+      });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -334,7 +360,8 @@ var TweetsClient = class {
    */
   async *getUserTweetsAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getUserTweets(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/latest_tweets`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -444,7 +471,8 @@ var UsersClient = class {
    */
   async *getFollowersAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getFollowers(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/followers`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -479,7 +507,8 @@ var UsersClient = class {
    */
   async *getFollowingAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getFollowing(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/followings`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -642,7 +671,8 @@ var UsersClient = class {
    */
   async *searchAll(query, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.search(query, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders("/v1/twitter/users/search_users", { params: { query, cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -702,7 +732,8 @@ var ListsClient = class {
    */
   async *getTweetsAll(listId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getTweets(listId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/lists/${listId}/tweets`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -737,7 +768,8 @@ var ListsClient = class {
    */
   async *getMembersAll(listId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getMembers(listId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/lists/${listId}/members`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -885,7 +917,14 @@ var CommunitiesClient = class {
    */
   async *getTweetsAll(communityId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getTweets(communityId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/communities/${communityId}/tweets`, {
+        params: {
+          tweet_type: options.tweetType ?? "Top",
+          count: options.count ?? 40,
+          cursor
+        }
+      });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }