scrapebadger 0.2.0 → 0.3.1

package/dist/twitter/index.d.cts

@@ -1,2 +1,2 @@
-export { _ as ApiResponse, ae as BillingLog, af as BillingLogList, j as CommunitiesClient, J as Community, E as CommunityBanner, K as CommunityMember, F as CommunityRule, p as CommunityTweetType, ag as ConnectOptions, a6 as ConnectedEvent, a3 as CreateMonitorParams, ac as DeliveryLog, ad as DeliveryLogList, a9 as ErrorEvent, aj as FilterRuleCreate, ao as FilterRuleDeliveryLog, ap as FilterRuleDeliveryLogListResponse, am as FilterRuleListResponse, al as FilterRulePricingTier, aq as FilterRulePricingTiersResponse, ai as FilterRuleResponse, ah as FilterRuleStatus, ak as FilterRuleUpdate, an as FilterRuleValidateResponse, G as GeoClient, l as GeoSearchOptions, H as Hashtag, D as List, $ as ListResponse, L as ListsClient, X as Location, M as Media, a0 as MonitorStatus, a7 as PingEvent, Z as Place, Y as PlaceTrends, r as Poll, q as PollOption, Q as QueryType, m as StreamClient, n as StreamEmitter, aa as StreamEvent, ab as StreamEventType, a1 as StreamMonitor, a2 as StreamMonitorList, a5 as StreamTweet, O as Trend, o as TrendCategory, k as TrendsClient, w as Tweet, a8 as TweetEvent, u as TweetPlace, i as TweetsClient, T as TwitterClient, a4 as UpdateMonitorParams, s as Url, x as User, y as UserAbout, z as UserIds, t as UserMention, U as UsersClient, v as verifyWebhookSignature } from '../index-BbCkdZXy.cjs';
+export { _ as ApiResponse, ae as BillingLog, af as BillingLogList, j as CommunitiesClient, J as Community, E as CommunityBanner, K as CommunityMember, F as CommunityRule, p as CommunityTweetType, ag as ConnectOptions, a6 as ConnectedEvent, a3 as CreateMonitorParams, ac as DeliveryLog, ad as DeliveryLogList, a9 as ErrorEvent, aj as FilterRuleCreate, ao as FilterRuleDeliveryLog, ap as FilterRuleDeliveryLogListResponse, am as FilterRuleListResponse, al as FilterRulePricingTier, aq as FilterRulePricingTiersResponse, ai as FilterRuleResponse, ah as FilterRuleStatus, ak as FilterRuleUpdate, an as FilterRuleValidateResponse, G as GeoClient, l as GeoSearchOptions, H as Hashtag, D as List, $ as ListResponse, L as ListsClient, X as Location, M as Media, a0 as MonitorStatus, a7 as PingEvent, Z as Place, Y as PlaceTrends, r as Poll, q as PollOption, Q as QueryType, m as StreamClient, n as StreamEmitter, aa as StreamEvent, ab as StreamEventType, a1 as StreamMonitor, a2 as StreamMonitorList, a5 as StreamTweet, O as Trend, o as TrendCategory, k as TrendsClient, w as Tweet, a8 as TweetEvent, u as TweetPlace, i as TweetsClient, T as TwitterClient, a4 as UpdateMonitorParams, s as Url, x as User, y as UserAbout, z as UserIds, t as UserMention, U as UsersClient, v as verifyWebhookSignature } from '../index-Cg0sNluO.cjs';
 import 'node:events';

package/dist/twitter/index.d.ts

@@ -1,2 +1,2 @@
-export { _ as ApiResponse, ae as BillingLog, af as BillingLogList, j as CommunitiesClient, J as Community, E as CommunityBanner, K as CommunityMember, F as CommunityRule, p as CommunityTweetType, ag as ConnectOptions, a6 as ConnectedEvent, a3 as CreateMonitorParams, ac as DeliveryLog, ad as DeliveryLogList, a9 as ErrorEvent, aj as FilterRuleCreate, ao as FilterRuleDeliveryLog, ap as FilterRuleDeliveryLogListResponse, am as FilterRuleListResponse, al as FilterRulePricingTier, aq as FilterRulePricingTiersResponse, ai as FilterRuleResponse, ah as FilterRuleStatus, ak as FilterRuleUpdate, an as FilterRuleValidateResponse, G as GeoClient, l as GeoSearchOptions, H as Hashtag, D as List, $ as ListResponse, L as ListsClient, X as Location, M as Media, a0 as MonitorStatus, a7 as PingEvent, Z as Place, Y as PlaceTrends, r as Poll, q as PollOption, Q as QueryType, m as StreamClient, n as StreamEmitter, aa as StreamEvent, ab as StreamEventType, a1 as StreamMonitor, a2 as StreamMonitorList, a5 as StreamTweet, O as Trend, o as TrendCategory, k as TrendsClient, w as Tweet, a8 as TweetEvent, u as TweetPlace, i as TweetsClient, T as TwitterClient, a4 as UpdateMonitorParams, s as Url, x as User, y as UserAbout, z as UserIds, t as UserMention, U as UsersClient, v as verifyWebhookSignature } from '../index-BbCkdZXy.js';
+export { _ as ApiResponse, ae as BillingLog, af as BillingLogList, j as CommunitiesClient, J as Community, E as CommunityBanner, K as CommunityMember, F as CommunityRule, p as CommunityTweetType, ag as ConnectOptions, a6 as ConnectedEvent, a3 as CreateMonitorParams, ac as DeliveryLog, ad as DeliveryLogList, a9 as ErrorEvent, aj as FilterRuleCreate, ao as FilterRuleDeliveryLog, ap as FilterRuleDeliveryLogListResponse, am as FilterRuleListResponse, al as FilterRulePricingTier, aq as FilterRulePricingTiersResponse, ai as FilterRuleResponse, ah as FilterRuleStatus, ak as FilterRuleUpdate, an as FilterRuleValidateResponse, G as GeoClient, l as GeoSearchOptions, H as Hashtag, D as List, $ as ListResponse, L as ListsClient, X as Location, M as Media, a0 as MonitorStatus, a7 as PingEvent, Z as Place, Y as PlaceTrends, r as Poll, q as PollOption, Q as QueryType, m as StreamClient, n as StreamEmitter, aa as StreamEvent, ab as StreamEventType, a1 as StreamMonitor, a2 as StreamMonitorList, a5 as StreamTweet, O as Trend, o as TrendCategory, k as TrendsClient, w as Tweet, a8 as TweetEvent, u as TweetPlace, i as TweetsClient, T as TwitterClient, a4 as UpdateMonitorParams, s as Url, x as User, y as UserAbout, z as UserIds, t as UserMention, U as UsersClient, v as verifyWebhookSignature } from '../index-Cg0sNluO.js';
 import 'node:events';

package/dist/twitter/index.js

@@ -16,12 +16,26 @@ function createPaginatedResponse(data, cursor) {
     hasMore: !!cursor
   };
 }
+var RATE_LIMIT_WARN_THRESHOLD = 0.2;
 async function* paginate(fetchPage, options = {}) {
   const { maxItems } = options;
   let cursor;
   let totalYielded = 0;
   do {
-    const response = await fetchPage(cursor);
+    const { response, rateLimit } = await fetchPage(cursor);
+    if (rateLimit) {
+      const { limit, remaining, reset } = rateLimit;
+      if (limit > 0 && remaining / limit < RATE_LIMIT_WARN_THRESHOLD) {
+        const nowSec = Date.now() / 1e3;
+        const windowRemainingSec = Math.max(reset - nowSec, 1);
+        const delayMs = remaining > 0 ? windowRemainingSec / remaining * 1e3 : windowRemainingSec * 1e3;
+        const resetInSec = Math.round(windowRemainingSec);
+        console.warn(
+          `\x1B[33m\u26A0 ScrapeBadger: Rate limit: ${remaining}/${limit} remaining (resets in ${resetInSec}s), throttling pagination\x1B[0m`
+        );
+        await sleep(delayMs);
+      }
+    }
     for (const item of response.data) {
       yield item;
       totalYielded++;
@@ -32,6 +46,9 @@ async function* paginate(fetchPage, options = {}) {
     cursor = response.nextCursor;
   } while (cursor);
 }
+function sleep(ms) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
 
 // src/twitter/tweets.ts
 var TweetsClient = class {
@@ -229,7 +246,8 @@ var TweetsClient = class {
    */
   async *getQuotesAll(tweetId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getQuotes(tweetId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/tweets/tweet/${tweetId}/quotes`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -296,7 +314,15 @@ var TweetsClient = class {
    */
   async *searchAll(query, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.search(query, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders("/v1/twitter/tweets/advanced_search", {
+        params: {
+          query,
+          query_type: options.queryType ?? "Top",
+          count: options.count,
+          cursor
+        }
+      });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -340,7 +366,8 @@ var TweetsClient = class {
    */
   async *getUserTweetsAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getUserTweets(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/latest_tweets`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -450,7 +477,8 @@ var UsersClient = class {
    */
   async *getFollowersAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getFollowers(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/followers`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -485,7 +513,8 @@ var UsersClient = class {
    */
   async *getFollowingAll(username, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getFollowing(username, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/users/${username}/followings`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -648,7 +677,8 @@ var UsersClient = class {
    */
   async *searchAll(query, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.search(query, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders("/v1/twitter/users/search_users", { params: { query, cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -708,7 +738,8 @@ var ListsClient = class {
    */
   async *getTweetsAll(listId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getTweets(listId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/lists/${listId}/tweets`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -743,7 +774,8 @@ var ListsClient = class {
    */
   async *getMembersAll(listId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getMembers(listId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/lists/${listId}/members`, { params: { cursor } });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }
@@ -891,7 +923,14 @@ var CommunitiesClient = class {
    */
   async *getTweetsAll(communityId, options = {}) {
     const fetchPage = async (cursor) => {
-      return this.getTweets(communityId, { ...options, cursor });
+      const { data, rateLimit } = await this.client.requestWithHeaders(`/v1/twitter/communities/${communityId}/tweets`, {
+        params: {
+          tweet_type: options.tweetType ?? "Top",
+          count: options.count ?? 40,
+          cursor
+        }
+      });
+      return { response: createPaginatedResponse(data.data ?? [], data.next_cursor), rateLimit };
     };
     yield* paginate(fetchPage, options);
   }