xbird-mcp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of xbird-mcp might be problematic. Click here for more details.

Files changed (85) hide show
  1. package/docs/listing.md +42 -0
  2. package/docs/mcp-setup.md +146 -0
  3. package/package.json +50 -0
  4. package/src/cli/context.ts +80 -0
  5. package/src/cli/output.ts +33 -0
  6. package/src/cli/program.ts +63 -0
  7. package/src/cli.ts +8 -0
  8. package/src/commands/about.ts +30 -0
  9. package/src/commands/bookmark.ts +40 -0
  10. package/src/commands/bookmarks-list.ts +108 -0
  11. package/src/commands/check.ts +27 -0
  12. package/src/commands/engagement.ts +31 -0
  13. package/src/commands/follow.ts +40 -0
  14. package/src/commands/followers.ts +132 -0
  15. package/src/commands/home.ts +59 -0
  16. package/src/commands/likes.ts +73 -0
  17. package/src/commands/lists.ts +101 -0
  18. package/src/commands/mentions.ts +32 -0
  19. package/src/commands/news.ts +38 -0
  20. package/src/commands/read.ts +36 -0
  21. package/src/commands/replies.ts +59 -0
  22. package/src/commands/search.ts +60 -0
  23. package/src/commands/tweet.ts +128 -0
  24. package/src/commands/user-tweets.ts +73 -0
  25. package/src/commands/user.ts +20 -0
  26. package/src/commands/whoami.ts +26 -0
  27. package/src/formatters/common.ts +49 -0
  28. package/src/formatters/list.ts +22 -0
  29. package/src/formatters/news.ts +23 -0
  30. package/src/formatters/tweet.ts +38 -0
  31. package/src/formatters/user.ts +20 -0
  32. package/src/index.ts +26 -0
  33. package/src/lib/auth.ts +105 -0
  34. package/src/lib/client-bookmarks.ts +62 -0
  35. package/src/lib/client-engagement.ts +152 -0
  36. package/src/lib/client-follow.ts +90 -0
  37. package/src/lib/client-full.ts +48 -0
  38. package/src/lib/client-likes.ts +62 -0
  39. package/src/lib/client-lists.ts +227 -0
  40. package/src/lib/client-media.ts +162 -0
  41. package/src/lib/client-news.ts +185 -0
  42. package/src/lib/client-posting.ts +163 -0
  43. package/src/lib/client-reading.ts +452 -0
  44. package/src/lib/client-search.ts +156 -0
  45. package/src/lib/client-timeline.ts +98 -0
  46. package/src/lib/client-user.ts +518 -0
  47. package/src/lib/client.ts +134 -0
  48. package/src/lib/config.ts +22 -0
  49. package/src/lib/constants.ts +55 -0
  50. package/src/lib/cookies.ts +132 -0
  51. package/src/lib/features.ts +175 -0
  52. package/src/lib/headers.ts +28 -0
  53. package/src/lib/paginate.ts +39 -0
  54. package/src/lib/query-ids.ts +190 -0
  55. package/src/lib/types.ts +147 -0
  56. package/src/lib/utils.ts +176 -0
  57. package/src/mcp/executor.ts +178 -0
  58. package/src/mcp/payment.ts +38 -0
  59. package/src/mcp/server.ts +53 -0
  60. package/src/mcp/tools.ts +389 -0
  61. package/src/server/app.ts +117 -0
  62. package/src/server/config/accounts.ts +137 -0
  63. package/src/server/config/pricing.ts +217 -0
  64. package/src/server/erc8004/register.ts +77 -0
  65. package/src/server/middleware/account-pool.ts +101 -0
  66. package/src/server/middleware/error-handler.ts +27 -0
  67. package/src/server/middleware/payer-extract.ts +43 -0
  68. package/src/server/middleware/x402.ts +61 -0
  69. package/src/server/routes/accounts.ts +93 -0
  70. package/src/server/routes/authorize.ts +19 -0
  71. package/src/server/routes/bookmarks.ts +21 -0
  72. package/src/server/routes/engagement.ts +84 -0
  73. package/src/server/routes/follow.ts +32 -0
  74. package/src/server/routes/health.ts +14 -0
  75. package/src/server/routes/lists.ts +38 -0
  76. package/src/server/routes/media.ts +44 -0
  77. package/src/server/routes/mentions.ts +20 -0
  78. package/src/server/routes/news.ts +23 -0
  79. package/src/server/routes/search.ts +26 -0
  80. package/src/server/routes/timeline.ts +21 -0
  81. package/src/server/routes/tweets.ts +82 -0
  82. package/src/server/routes/users.ts +92 -0
  83. package/src/server/storage/accounts-db.ts +84 -0
  84. package/src/server.ts +34 -0
  85. package/tsconfig.json +29 -0
@@ -0,0 +1,55 @@
1
+ export const BEARER_TOKEN =
2
+ "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
3
+
4
+ export const TWITTER_API_BASE = "https://x.com/i/api/graphql";
5
+ export const TWITTER_V1_BASE = "https://x.com/i/api/1.1";
6
+ export const TWITTER_UPLOAD_URL =
7
+ "https://upload.twitter.com/i/media/upload.json";
8
+
9
+ export const DEFAULT_USER_AGENT =
10
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
11
+
12
+ export const CACHE_DIR = `${process.env.HOME ?? "~"}/.config/xbird`;
13
+ export const QUERY_IDS_CACHE_FILE = `${CACHE_DIR}/query-ids-cache.json`;
14
+ export const CACHE_TTL_MS = 24 * 60 * 60 * 1000; // 24h
15
+
16
+ /** Fallback query IDs — updated periodically, used when cache is empty */
17
+ export const FALLBACK_QUERY_IDS: Record<string, string> = {
18
+ CreateTweet: "nmdAQXJDxw6-0KKF2on7eA",
19
+ CreateRetweet: "LFho5rIi4xcKO90p9jwG7A",
20
+ DeleteRetweet: "iQtK4dl5hBmXewYZuEOKVw",
21
+ CreateFriendship: "8h9JVdV8dlSyqyRDJEPCsA",
22
+ DestroyFriendship: "ppXWuagMNXgvzx6WoXBW0Q",
23
+ FavoriteTweet: "lI07N6Otwv1PhnEgXILM7A",
24
+ UnfavoriteTweet: "ZYKSe-w7KEslx3JhSIk5LA",
25
+ CreateBookmark: "aoDbu3RHznuiSkQ9aNM67Q",
26
+ DeleteBookmark: "Wlmlj2-xzyS1GN3a6cj-mQ",
27
+ TweetDetail: "_NvJCnIjOW__EP5-RF197A",
28
+ SearchTimeline: "6AAys3t42mosm_yTI_QENg",
29
+ HomeTimeline: "edseUwk9sP5Phz__9TIRnA",
30
+ HomeLatestTimeline: "iOEZpOdfekFsxSlPQCQtPg",
31
+ UserByScreenName: "xmU6X_CKcnQ5lSrCbAmJsg",
32
+ Viewer: "pVrmNaXcxPjisIvKtLDMEA",
33
+ UserTweets: "Wms1GvIiHXAPBaCr9KblaA",
34
+ Bookmarks: "RV1g3b8n_SGOHwkqKYSCFw",
35
+ Following: "mWYeougg_ocJS2Vr1Vt28w",
36
+ Followers: "SFYY3WsgwjlXSLlfnEUE4A",
37
+ Likes: "ETJflBunfqNa1uE1mBPCaw",
38
+ ListOwnerships: "wQcOSjSQ8NtgxIwvYl1lMg",
39
+ ListMemberships: "BlEXXdARdSeL_0KyKHHvvg",
40
+ ListLatestTweetsTimeline: "2TemLyqrMpTeAmysdbnVqw",
41
+ GenericTimelineById: "SpoeI4hKtSMEjhBjQl8rFA",
42
+ };
43
+
44
+ export const TARGET_OPERATIONS = Object.keys(FALLBACK_QUERY_IDS);
45
+
46
+ /** Bundle URL regex for discovering query IDs from Twitter JS bundles */
47
+ export const BUNDLE_URL_REGEX =
48
+ /https:\/\/abs\.twimg\.com\/responsive-web\/client-web(?:-legacy)?\/[A-Za-z0-9.-]+\.js/g;
49
+
50
+ /** Discovery pages to scrape for bundle URLs */
51
+ export const DISCOVERY_PAGES = [
52
+ "https://x.com/?lang=en",
53
+ "https://x.com/explore",
54
+ "https://x.com/notifications",
55
+ ];
@@ -0,0 +1,132 @@
1
+ import { getCookies, toCookieHeader } from "@steipete/sweet-cookie";
2
+ import type { BrowserName } from "@steipete/sweet-cookie";
3
+ import type { ExtendedBrowserName } from "./types.ts";
4
+
5
+ const TWITTER_URL = "https://x.com/";
6
+ const TWITTER_ORIGINS = ["https://x.com/", "https://twitter.com/"];
7
+ const TWITTER_COOKIE_NAMES = ["auth_token", "ct0"];
8
+
9
+ const CHROMIUM_PROFILES: Record<string, string> = {
10
+ arc: `${process.env.HOME}/Library/Application Support/Arc/User Data/Default`,
11
+ brave: `${process.env.HOME}/Library/Application Support/BraveSoftware/Brave-Browser/Default`,
12
+ };
13
+
14
+ export interface BrowserMapping {
15
+ browserName: BrowserName;
16
+ chromeProfile?: string;
17
+ }
18
+
19
+ export function mapBrowserName(source?: ExtendedBrowserName): BrowserMapping | undefined {
20
+ if (!source) return undefined;
21
+ if (source === "arc" || source === "brave") {
22
+ return { browserName: "chrome", chromeProfile: CHROMIUM_PROFILES[source] };
23
+ }
24
+ return { browserName: source as BrowserName };
25
+ }
26
+
27
+ export interface BrowserCookieResult {
28
+ authToken: string | null;
29
+ ct0: string | null;
30
+ source: string | null;
31
+ warnings: string[];
32
+ }
33
+
34
+ function pickValue(
35
+ cookies: Array<{ name: string; value: string; domain?: string }>,
36
+ name: string
37
+ ): string | null {
38
+ const matches = cookies.filter(
39
+ (c) => c.name === name && typeof c.value === "string" && c.value.length > 0
40
+ );
41
+ if (matches.length === 0) return null;
42
+ const preferred = matches.find((c) => (c.domain ?? "").endsWith("x.com"));
43
+ if (preferred) return preferred.value;
44
+ const twitter = matches.find((c) =>
45
+ (c.domain ?? "").endsWith("twitter.com")
46
+ );
47
+ if (twitter) return twitter.value;
48
+ return matches[0]?.value ?? null;
49
+ }
50
+
51
+ interface BrowserAttempt {
52
+ browser: BrowserName;
53
+ chromeProfile?: string;
54
+ displayName?: string;
55
+ }
56
+
57
+ const DEFAULT_ATTEMPTS: BrowserAttempt[] = [
58
+ { browser: "safari" },
59
+ { browser: "chrome" },
60
+ { browser: "chrome", chromeProfile: CHROMIUM_PROFILES.arc, displayName: "Arc" },
61
+ { browser: "chrome", chromeProfile: CHROMIUM_PROFILES.brave, displayName: "Brave" },
62
+ { browser: "firefox" },
63
+ ];
64
+
65
+ export async function extractBrowserCookies(
66
+ browsers?: BrowserName[],
67
+ chromeProfile?: string,
68
+ ): Promise<BrowserCookieResult> {
69
+ const warnings: string[] = [];
70
+
71
+ // If specific browsers provided, use them directly
72
+ if (browsers) {
73
+ for (const browser of browsers) {
74
+ try {
75
+ const { cookies, warnings: w } = await getCookies({
76
+ url: TWITTER_URL,
77
+ origins: TWITTER_ORIGINS,
78
+ names: [...TWITTER_COOKIE_NAMES],
79
+ browsers: [browser],
80
+ mode: "merge",
81
+ timeoutMs: 30_000,
82
+ ...(chromeProfile && browser === "chrome" ? { chromeProfile } : {}),
83
+ } as Parameters<typeof getCookies>[0]);
84
+ warnings.push(...w);
85
+
86
+ const authToken = pickValue(cookies as Array<{ name: string; value: string; domain?: string }>, "auth_token");
87
+ const ct0 = pickValue(cookies as Array<{ name: string; value: string; domain?: string }>, "ct0");
88
+
89
+ if (authToken && ct0) {
90
+ return {
91
+ authToken,
92
+ ct0,
93
+ source: browser.charAt(0).toUpperCase() + browser.slice(1),
94
+ warnings,
95
+ };
96
+ }
97
+ } catch {
98
+ warnings.push(`Failed to read cookies from ${browser}`);
99
+ }
100
+ }
101
+ return { authToken: null, ct0: null, source: null, warnings };
102
+ }
103
+
104
+ // Default: try multiple browsers including Arc/Brave
105
+ for (const attempt of DEFAULT_ATTEMPTS) {
106
+ try {
107
+ const { cookies, warnings: w } = await getCookies({
108
+ url: TWITTER_URL,
109
+ origins: TWITTER_ORIGINS,
110
+ names: [...TWITTER_COOKIE_NAMES],
111
+ browsers: [attempt.browser],
112
+ mode: "merge",
113
+ timeoutMs: 30_000,
114
+ ...(attempt.chromeProfile ? { chromeProfile: attempt.chromeProfile } : {}),
115
+ } as Parameters<typeof getCookies>[0]);
116
+ warnings.push(...w);
117
+
118
+ const authToken = pickValue(cookies as Array<{ name: string; value: string; domain?: string }>, "auth_token");
119
+ const ct0 = pickValue(cookies as Array<{ name: string; value: string; domain?: string }>, "ct0");
120
+
121
+ if (authToken && ct0) {
122
+ const displayName = attempt.displayName ?? (attempt.browser.charAt(0).toUpperCase() + attempt.browser.slice(1));
123
+ return { authToken, ct0, source: displayName, warnings };
124
+ }
125
+ } catch {
126
+ const name = attempt.displayName ?? attempt.browser;
127
+ warnings.push(`Failed to read cookies from ${name}`);
128
+ }
129
+ }
130
+
131
+ return { authToken: null, ct0: null, source: null, warnings };
132
+ }
@@ -0,0 +1,175 @@
1
+ /** Feature flags for TweetDetail requests */
2
+ export function tweetDetailFeatures(): Record<string, boolean> {
3
+ return {
4
+ rweb_video_screen_enabled: true,
5
+ profile_label_improvements_pcf_label_in_post_enabled: true,
6
+ responsive_web_profile_redirect_enabled: true,
7
+ rweb_tipjar_consumption_enabled: true,
8
+ verified_phone_label_enabled: false,
9
+ creator_subscriptions_tweet_preview_api_enabled: true,
10
+ responsive_web_graphql_timeline_navigation_enabled: true,
11
+ responsive_web_graphql_exclude_directive_enabled: true,
12
+ responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
13
+ premium_content_api_read_enabled: false,
14
+ communities_web_enable_tweet_community_results_fetch: true,
15
+ c9s_tweet_anatomy_moderator_badge_enabled: true,
16
+ responsive_web_grok_analyze_button_fetch_trends_enabled: false,
17
+ responsive_web_grok_analyze_post_followups_enabled: false,
18
+ responsive_web_grok_annotations_enabled: false,
19
+ responsive_web_jetfuel_frame: true,
20
+ post_ctas_fetch_enabled: true,
21
+ responsive_web_grok_share_attachment_enabled: true,
22
+ articles_preview_enabled: true,
23
+ responsive_web_edit_tweet_api_enabled: true,
24
+ graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
25
+ view_counts_everywhere_api_enabled: true,
26
+ longform_notetweets_consumption_enabled: true,
27
+ responsive_web_twitter_article_tweet_consumption_enabled: true,
28
+ tweet_awards_web_tipping_enabled: false,
29
+ responsive_web_grok_show_grok_translated_post: false,
30
+ responsive_web_grok_analysis_button_from_backend: true,
31
+ creator_subscriptions_quote_tweet_preview_enabled: false,
32
+ freedom_of_speech_not_reach_fetch_enabled: true,
33
+ standardized_nudges_misinfo: true,
34
+ tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
35
+ longform_notetweets_rich_text_read_enabled: true,
36
+ longform_notetweets_inline_media_enabled: true,
37
+ responsive_web_grok_image_annotation_enabled: true,
38
+ responsive_web_grok_imagine_annotation_enabled: true,
39
+ responsive_web_grok_community_note_auto_translation_is_enabled: false,
40
+ responsive_web_enhance_cards_enabled: false,
41
+ responsive_web_twitter_article_plain_text_enabled: true,
42
+ responsive_web_twitter_article_seed_tweet_detail_enabled: true,
43
+ responsive_web_twitter_article_seed_tweet_summary_enabled: true,
44
+ rweb_video_timestamps_enabled: true,
45
+ subscriptions_feature_can_gift_premium: false,
46
+ subscriptions_verification_info_is_identity_verified_enabled: true,
47
+ highlights_tweets_tab_ui_enabled: true,
48
+ subscriptions_verification_info_verified_since_enabled: true,
49
+ responsive_web_twitter_article_notes_tab_enabled: true,
50
+ hidden_profile_subscriptions_enabled: true,
51
+ };
52
+ }
53
+
54
+ /** Feature flags for CreateTweet requests */
55
+ export function createTweetFeatures(): Record<string, boolean> {
56
+ return {
57
+ rweb_video_screen_enabled: true,
58
+ creator_subscriptions_tweet_preview_api_enabled: true,
59
+ premium_content_api_read_enabled: false,
60
+ communities_web_enable_tweet_community_results_fetch: true,
61
+ c9s_tweet_anatomy_moderator_badge_enabled: true,
62
+ responsive_web_grok_analyze_button_fetch_trends_enabled: false,
63
+ responsive_web_grok_analyze_post_followups_enabled: false,
64
+ responsive_web_grok_annotations_enabled: false,
65
+ responsive_web_jetfuel_frame: true,
66
+ post_ctas_fetch_enabled: true,
67
+ responsive_web_grok_share_attachment_enabled: true,
68
+ responsive_web_edit_tweet_api_enabled: true,
69
+ graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
70
+ view_counts_everywhere_api_enabled: true,
71
+ longform_notetweets_consumption_enabled: true,
72
+ responsive_web_twitter_article_tweet_consumption_enabled: true,
73
+ tweet_awards_web_tipping_enabled: false,
74
+ responsive_web_grok_show_grok_translated_post: false,
75
+ responsive_web_grok_analysis_button_from_backend: true,
76
+ creator_subscriptions_quote_tweet_preview_enabled: false,
77
+ longform_notetweets_rich_text_read_enabled: true,
78
+ longform_notetweets_inline_media_enabled: true,
79
+ profile_label_improvements_pcf_label_in_post_enabled: true,
80
+ responsive_web_profile_redirect_enabled: false,
81
+ rweb_tipjar_consumption_enabled: true,
82
+ verified_phone_label_enabled: false,
83
+ articles_preview_enabled: true,
84
+ responsive_web_grok_community_note_auto_translation_is_enabled: false,
85
+ responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
86
+ freedom_of_speech_not_reach_fetch_enabled: true,
87
+ standardized_nudges_misinfo: true,
88
+ tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
89
+ responsive_web_grok_image_annotation_enabled: true,
90
+ responsive_web_grok_imagine_annotation_enabled: true,
91
+ responsive_web_graphql_timeline_navigation_enabled: true,
92
+ responsive_web_enhance_cards_enabled: false,
93
+ };
94
+ }
95
+
96
+ /** Feature flags for SearchTimeline requests */
97
+ export function searchFeatures(): Record<string, boolean> {
98
+ return {
99
+ ...tweetDetailFeatures(),
100
+ rweb_video_timestamps_enabled: true,
101
+ };
102
+ }
103
+
104
+ /** Feature flags for HomeTimeline requests */
105
+ export function timelineFeatures(): Record<string, boolean> {
106
+ return {
107
+ ...searchFeatures(),
108
+ blue_business_profile_image_shape_enabled: true,
109
+ responsive_web_text_conversations_enabled: false,
110
+ tweetypie_unmention_optimization_enabled: true,
111
+ vibe_api_enabled: true,
112
+ responsive_web_twitter_blue_verified_badge_is_enabled: true,
113
+ interactive_text_enabled: true,
114
+ longform_notetweets_richtext_consumption_enabled: true,
115
+ responsive_web_media_download_video_enabled: false,
116
+ };
117
+ }
118
+
119
+ /** Feature flags for Viewer (whoami) requests */
120
+ export function viewerFeatures(): Record<string, boolean> {
121
+ return {
122
+ ...tweetDetailFeatures(),
123
+ subscriptions_upsells_api_enabled: true,
124
+ };
125
+ }
126
+
127
+ /** Field toggles for tweet detail */
128
+ export function fieldToggles(): Record<string, boolean> {
129
+ return {
130
+ withArticleRichContentState: true,
131
+ withArticlePlainText: true,
132
+ withGrokAnalyze: false,
133
+ withDisallowedReplyControls: false,
134
+ };
135
+ }
136
+
137
+ /** Feature flags for List operations */
138
+ export function listFeatures(): Record<string, boolean> {
139
+ return {
140
+ ...tweetDetailFeatures(),
141
+ rweb_video_timestamps_enabled: true,
142
+ };
143
+ }
144
+
145
+ /** Feature flags for News/Explore (GenericTimelineById) */
146
+ export function newsFeatures(): Record<string, boolean> {
147
+ return {
148
+ ...tweetDetailFeatures(),
149
+ rweb_video_timestamps_enabled: true,
150
+ };
151
+ }
152
+
153
+ /** Feature flags for Bookmarks */
154
+ export function bookmarksFeatures(): Record<string, boolean> {
155
+ return {
156
+ ...timelineFeatures(),
157
+ graphql_timeline_v2_bookmark_timeline: true,
158
+ };
159
+ }
160
+
161
+ /** Feature flags for Likes */
162
+ export function likesFeatures(): Record<string, boolean> {
163
+ return {
164
+ ...tweetDetailFeatures(),
165
+ rweb_video_timestamps_enabled: true,
166
+ };
167
+ }
168
+
169
+ /** Feature flags for UserTweets */
170
+ export function userTweetsFeatures(): Record<string, boolean> {
171
+ return {
172
+ ...tweetDetailFeatures(),
173
+ rweb_video_timestamps_enabled: true,
174
+ };
175
+ }
@@ -0,0 +1,28 @@
1
+ import { randomBytes, randomUUID } from "node:crypto";
2
+ import { BEARER_TOKEN, DEFAULT_USER_AGENT } from "./constants.ts";
3
+ import type { Credentials } from "./types.ts";
4
+
5
+ export function buildHeaders(creds: Credentials): Record<string, string> {
6
+ return {
7
+ accept: "*/*",
8
+ "accept-language": "en-US,en;q=0.9",
9
+ authorization: `Bearer ${BEARER_TOKEN}`,
10
+ "x-csrf-token": creds.ct0,
11
+ "x-twitter-auth-type": "OAuth2Session",
12
+ "x-twitter-active-user": "yes",
13
+ "x-twitter-client-language": "en",
14
+ "x-client-uuid": randomUUID(),
15
+ "x-client-transaction-id": randomBytes(16).toString("hex"),
16
+ cookie: `auth_token=${creds.authToken}; ct0=${creds.ct0}`,
17
+ "user-agent": DEFAULT_USER_AGENT,
18
+ origin: "https://x.com",
19
+ referer: "https://x.com/",
20
+ "content-type": "application/json",
21
+ };
22
+ }
23
+
24
+ export function buildBaseHeaders(creds: Credentials): Record<string, string> {
25
+ const headers = buildHeaders(creds);
26
+ delete headers["content-type"];
27
+ return headers;
28
+ }
@@ -0,0 +1,39 @@
1
+ export interface AutoPaginateOpts {
2
+ all?: boolean;
3
+ maxPages?: number;
4
+ delay?: number;
5
+ }
6
+
7
+ export async function autoPaginate<T>(
8
+ fetchPage: (cursor?: string) => Promise<{ items: T[]; cursor?: string }>,
9
+ opts: AutoPaginateOpts = {}
10
+ ): Promise<{ items: T[]; cursor?: string }> {
11
+ const allItems: T[] = [];
12
+ let cursor: string | undefined;
13
+ let pages = 0;
14
+ const seenCursors = new Set<string>();
15
+
16
+ // If not paginating, just fetch one page
17
+ if (!opts.all && !opts.maxPages) {
18
+ const result = await fetchPage();
19
+ return result;
20
+ }
21
+
22
+ const maxPages = opts.all ? Infinity : (opts.maxPages ?? 1);
23
+
24
+ while (pages < maxPages) {
25
+ const result = await fetchPage(cursor);
26
+ allItems.push(...result.items);
27
+ pages++;
28
+
29
+ if (!result.cursor || seenCursors.has(result.cursor)) break;
30
+ seenCursors.add(result.cursor);
31
+ cursor = result.cursor;
32
+
33
+ if (pages < maxPages && opts.delay) {
34
+ await new Promise(r => setTimeout(r, opts.delay));
35
+ }
36
+ }
37
+
38
+ return { items: allItems, cursor };
39
+ }
@@ -0,0 +1,190 @@
1
+ import { mkdir, readFile, writeFile } from "node:fs/promises";
2
+ import path from "node:path";
3
+ import {
4
+ BUNDLE_URL_REGEX,
5
+ CACHE_TTL_MS,
6
+ DISCOVERY_PAGES,
7
+ FALLBACK_QUERY_IDS,
8
+ QUERY_IDS_CACHE_FILE,
9
+ TARGET_OPERATIONS,
10
+ } from "./constants.ts";
11
+
12
+ interface CacheSnapshot {
13
+ fetchedAt: string;
14
+ ttlMs: number;
15
+ ids: Record<string, string>;
16
+ }
17
+
18
+ const OPERATION_PATTERNS = [
19
+ {
20
+ regex:
21
+ /e\.exports=\{queryId\s*:\s*["']([^"']+)["']\s*,\s*operationName\s*:\s*["']([^"']+)["']/gs,
22
+ opGroup: 2,
23
+ idGroup: 1,
24
+ },
25
+ {
26
+ regex:
27
+ /e\.exports=\{operationName\s*:\s*["']([^"']+)["']\s*,\s*queryId\s*:\s*["']([^"']+)["']/gs,
28
+ opGroup: 1,
29
+ idGroup: 2,
30
+ },
31
+ {
32
+ regex:
33
+ /operationName\s*[:=]\s*["']([^"']+)["'](.{0,4000}?)queryId\s*[:=]\s*["']([^"']+)["']/gs,
34
+ opGroup: 1,
35
+ idGroup: 3,
36
+ },
37
+ {
38
+ regex:
39
+ /queryId\s*[:=]\s*["']([^"']+)["'](.{0,4000}?)operationName\s*[:=]\s*["']([^"']+)["']/gs,
40
+ opGroup: 3,
41
+ idGroup: 1,
42
+ },
43
+ ];
44
+
45
+ const QUERY_ID_REGEX = /^[a-zA-Z0-9_-]+$/;
46
+
47
+ let memoryCache: CacheSnapshot | null = null;
48
+ let loadPromise: Promise<CacheSnapshot | null> | null = null;
49
+
50
+ async function readCache(): Promise<CacheSnapshot | null> {
51
+ try {
52
+ const raw = await readFile(QUERY_IDS_CACHE_FILE, "utf8");
53
+ const parsed = JSON.parse(raw);
54
+ if (parsed?.fetchedAt && parsed?.ids) {
55
+ return parsed as CacheSnapshot;
56
+ }
57
+ } catch {
58
+ // no cache yet
59
+ }
60
+ return null;
61
+ }
62
+
63
+ async function writeCache(snapshot: CacheSnapshot): Promise<void> {
64
+ await mkdir(path.dirname(QUERY_IDS_CACHE_FILE), { recursive: true });
65
+ await writeFile(
66
+ QUERY_IDS_CACHE_FILE,
67
+ JSON.stringify(snapshot, null, 2) + "\n",
68
+ "utf8"
69
+ );
70
+ }
71
+
72
+ function isFresh(snapshot: CacheSnapshot): boolean {
73
+ const age = Date.now() - new Date(snapshot.fetchedAt).getTime();
74
+ return age <= (snapshot.ttlMs || CACHE_TTL_MS);
75
+ }
76
+
77
+ async function loadSnapshot(): Promise<CacheSnapshot | null> {
78
+ if (memoryCache) return memoryCache;
79
+ if (!loadPromise) {
80
+ loadPromise = readCache().then((s) => {
81
+ memoryCache = s;
82
+ return s;
83
+ });
84
+ }
85
+ return loadPromise;
86
+ }
87
+
88
+ /**
89
+ * Get a query ID for the given operation name.
90
+ * Checks cache first, then falls back to hardcoded IDs.
91
+ */
92
+ export async function getQueryId(operationName: string): Promise<string> {
93
+ const snapshot = await loadSnapshot();
94
+ const cached = snapshot?.ids[operationName];
95
+ if (cached) return cached;
96
+ return FALLBACK_QUERY_IDS[operationName] ?? operationName;
97
+ }
98
+
99
+ /**
100
+ * Discover fresh query IDs from Twitter's JS bundles.
101
+ */
102
+ export async function refreshQueryIds(
103
+ options: { force?: boolean } = {}
104
+ ): Promise<void> {
105
+ const snapshot = await loadSnapshot();
106
+ if (!options.force && snapshot && isFresh(snapshot)) return;
107
+
108
+ const targets = new Set(TARGET_OPERATIONS);
109
+ const discovered = new Map<string, string>();
110
+
111
+ // Step 1: Discover bundle URLs from Twitter HTML pages
112
+ const bundleUrls = new Set<string>();
113
+ for (const page of DISCOVERY_PAGES) {
114
+ try {
115
+ const res = await fetch(page, {
116
+ headers: {
117
+ "User-Agent":
118
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
119
+ Accept: "text/html",
120
+ },
121
+ });
122
+ if (!res.ok) continue;
123
+ const html = await res.text();
124
+ for (const match of html.matchAll(BUNDLE_URL_REGEX)) {
125
+ bundleUrls.add(match[0]);
126
+ }
127
+ } catch {
128
+ // ignore individual page failures
129
+ }
130
+ }
131
+
132
+ if (bundleUrls.size === 0) return;
133
+
134
+ // Step 2: Fetch bundles and extract query IDs (concurrency: 6)
135
+ const urls = [...bundleUrls];
136
+ const CONCURRENCY = 6;
137
+ for (let i = 0; i < urls.length && discovered.size < targets.size; i += CONCURRENCY) {
138
+ const chunk = urls.slice(i, i + CONCURRENCY);
139
+ await Promise.all(
140
+ chunk.map(async (url) => {
141
+ if (discovered.size >= targets.size) return;
142
+ try {
143
+ const res = await fetch(url);
144
+ if (!res.ok) return;
145
+ const js = await res.text();
146
+ for (const pattern of OPERATION_PATTERNS) {
147
+ pattern.regex.lastIndex = 0;
148
+ let match: RegExpExecArray | null;
149
+ while ((match = pattern.regex.exec(js)) !== null) {
150
+ const op = match[pattern.opGroup];
151
+ const id = match[pattern.idGroup];
152
+ if (!op || !id || !targets.has(op) || discovered.has(op)) continue;
153
+ if (!QUERY_ID_REGEX.test(id)) continue;
154
+ discovered.set(op, id);
155
+ }
156
+ }
157
+ } catch {
158
+ // ignore individual bundle failures
159
+ }
160
+ })
161
+ );
162
+ }
163
+
164
+ if (discovered.size === 0) return;
165
+
166
+ // Step 3: Save to cache
167
+ const ids: Record<string, string> = {};
168
+ for (const name of TARGET_OPERATIONS) {
169
+ const id = discovered.get(name);
170
+ if (id) ids[name] = id;
171
+ }
172
+
173
+ const newSnapshot: CacheSnapshot = {
174
+ fetchedAt: new Date().toISOString(),
175
+ ttlMs: CACHE_TTL_MS,
176
+ ids,
177
+ };
178
+
179
+ await writeCache(newSnapshot);
180
+ memoryCache = newSnapshot;
181
+ loadPromise = null;
182
+ }
183
+
184
+ /**
185
+ * Clear in-memory cache (useful for tests).
186
+ */
187
+ export function clearCache(): void {
188
+ memoryCache = null;
189
+ loadPromise = null;
190
+ }