rsshub 1.0.0-master.f9003f9 → 1.0.0-master.f917bdc

package/lib/routes/twitter/api/mobile-api/login.ts

@@ -4,16 +4,13 @@ import { bearerToken, guestActivateUrl } from './constants';
 import got from '@/utils/got';
 import ofetch from '@/utils/ofetch';
 import crypto from 'crypto';
-import { config } from '@/config';
 import { v5 as uuidv5 } from 'uuid';
 import { authenticator } from 'otplib';
 import logger from '@/utils/logger';
 import cache from '@/utils/cache';
+import { RateLimiterMemory, RateLimiterRedis, RateLimiterQueue } from 'rate-limiter-flexible';
 
 const NAMESPACE = 'd41d092b-b007-48f7-9129-e9538d2d8fe9';
-const username = config.twitter.username;
-const password = config.twitter.password;
-const authenticationSecret = config.twitter.authenticationSecret;
 
 let authentication = null;
 
@@ -29,157 +26,183 @@ const headers = {
     Authorization: bearerToken,
 };
 
-async function login() {
-    return await cache.tryGet(
+const loginLimiter = cache.clients.redisClient
+    ? new RateLimiterRedis({
+          points: 1,
+          duration: 20,
+          execEvenly: true,
+          storeClient: cache.clients.redisClient,
+      })
+    : new RateLimiterMemory({
+          points: 1,
+          duration: 20,
+          execEvenly: true,
+      });
+
+const loginLimiterQueue = new RateLimiterQueue(loginLimiter);
+
+async function login({ username, password, authenticationSecret }) {
+    return (await cache.tryGet(
         `twitter:authentication:${username}`,
         async () => {
-            logger.debug('Twitter login start.');
-            const android_id = uuidv5(username, NAMESPACE);
-            headers['X-Twitter-Client-DeviceID'] = android_id;
-
-            const ct0 = crypto.randomUUID().replaceAll('-', '');
-            const guestToken = await got(guestActivateUrl, {
-                headers: {
-                    authorization: bearerToken,
-                    'x-csrf-token': ct0,
-                    cookie: 'ct0=' + ct0,
-                },
-                method: 'POST',
-            });
-            logger.debug('Twitter login 1 finished: guest token.');
-
-            headers['x-guest-token'] = guestToken.data.guest_token;
-
-            const task1 = await ofetch.raw(
-                'https://api.x.com/1.1/onboarding/task.json?' +
-                    new URLSearchParams({
-                        flow_name: 'login',
-                        api_version: '1',
-                        known_device_token: '',
-                        sim_country_code: 'us',
-                    }).toString(),
-                {
+            try {
+                await loginLimiterQueue.removeTokens(1);
+
+                logger.debug('Twitter login start.');
+                const android_id = uuidv5(username, NAMESPACE);
+                headers['X-Twitter-Client-DeviceID'] = android_id;
+
+                const ct0 = crypto.randomUUID().replaceAll('-', '');
+                const guestToken = await got(guestActivateUrl, {
+                    headers: {
+                        authorization: bearerToken,
+                        'x-csrf-token': ct0,
+                        cookie: 'ct0=' + ct0,
+                    },
                     method: 'POST',
-                    headers,
-                    body: {
-                        flow_token: null,
-                        input_flow_data: {
-                            country_code: null,
-                            flow_context: {
-                                referrer_context: {
-                                    referral_details: 'utm_source=google-play&utm_medium=organic',
-                                    referrer_url: '',
-                                },
-                                start_location: {
-                                    location: 'deeplink',
+                });
+                logger.debug('Twitter login 1 finished: guest token.');
+
+                headers['x-guest-token'] = guestToken.data.guest_token;
+
+                const task1 = await ofetch.raw(
+                    'https://api.x.com/1.1/onboarding/task.json?' +
+                        new URLSearchParams({
+                            flow_name: 'login',
+                            api_version: '1',
+                            known_device_token: '',
+                            sim_country_code: 'us',
+                        }).toString(),
+                    {
+                        method: 'POST',
+                        headers,
+                        body: {
+                            flow_token: null,
+                            input_flow_data: {
+                                country_code: null,
+                                flow_context: {
+                                    referrer_context: {
+                                        referral_details: 'utm_source=google-play&utm_medium=organic',
+                                        referrer_url: '',
+                                    },
+                                    start_location: {
+                                        location: 'deeplink',
+                                    },
                                 },
+                                requested_variant: null,
+                                target_user_id: 0,
                             },
-                            requested_variant: null,
-                            target_user_id: 0,
                         },
-                    },
-                }
-            );
-            logger.debug('Twitter login 2 finished: login flow.');
-
-            headers.att = task1.headers.get('att');
-
-            const task2 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
-                headers,
-                json: {
-                    flow_token: task1._data.flow_token,
-                    subtask_inputs: [
-                        {
-                            enter_text: {
-                                suggestion_id: null,
-                                text: username,
-                                link: 'next_link',
+                    }
+                );
+                logger.debug('Twitter login 2 finished: login flow.');
+
+                headers.att = task1.headers.get('att');
+
+                const task2 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
+                    headers,
+                    json: {
+                        flow_token: task1._data.flow_token,
+                        subtask_inputs: [
+                            {
+                                enter_text: {
+                                    suggestion_id: null,
+                                    text: username,
+                                    link: 'next_link',
+                                },
+                                subtask_id: 'LoginEnterUserIdentifier',
                             },
-                            subtask_id: 'LoginEnterUserIdentifier',
-                        },
-                    ],
-                },
-            });
-            logger.debug('Twitter login 3 finished: LoginEnterUserIdentifier.');
-
-            const task3 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
-                headers,
-                json: {
-                    flow_token: task2.data.flow_token,
-                    subtask_inputs: [
-                        {
-                            enter_password: {
-                                password,
-                                link: 'next_link',
+                        ],
+                    },
+                });
+                logger.debug('Twitter login 3 finished: LoginEnterUserIdentifier.');
+
+                const task3 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
+                    headers,
+                    json: {
+                        flow_token: task2.data.flow_token,
+                        subtask_inputs: [
+                            {
+                                enter_password: {
+                                    password,
+                                    link: 'next_link',
+                                },
+                                subtask_id: 'LoginEnterPassword',
                             },
-                            subtask_id: 'LoginEnterPassword',
-                        },
-                    ],
-                },
-            });
-            logger.debug('Twitter login 4 finished: LoginEnterPassword.');
-
-            const task4 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
-                headers,
-                json: {
-                    flow_token: task3.data.flow_token,
-                    subtask_inputs: [
-                        {
-                            check_logged_in_account: {
-                                link: 'AccountDuplicationCheck_false',
+                        ],
+                    },
+                });
+                logger.debug('Twitter login 4 finished: LoginEnterPassword.');
+
+                const task4 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
+                    headers,
+                    json: {
+                        flow_token: task3.data.flow_token,
+                        subtask_inputs: [
+                            {
+                                check_logged_in_account: {
+                                    link: 'AccountDuplicationCheck_false',
+                                },
+                                subtask_id: 'AccountDuplicationCheck',
                             },
-                            subtask_id: 'AccountDuplicationCheck',
-                        },
-                    ],
-                },
-            });
-            logger.debug('Twitter login 5 finished: AccountDuplicationCheck.');
-
-            for await (const subtask of task4.data?.subtasks || []) {
-                if (subtask.open_account) {
-                    authentication = subtask.open_account;
-                    break;
-                } else if (subtask.subtask_id === 'LoginTwoFactorAuthChallenge') {
-                    const token = authenticator.generate(authenticationSecret);
-
-                    const task5 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
-                        headers,
-                        json: {
-                            flow_token: task4.data.flow_token,
-                            subtask_inputs: [
-                                {
-                                    enter_text: {
-                                        suggestion_id: null,
-                                        text: token,
-                                        link: 'next_link',
+                        ],
+                    },
+                });
+                logger.debug('Twitter login 5 finished: AccountDuplicationCheck.');
+
+                for await (const subtask of task4.data?.subtasks || []) {
+                    if (subtask.open_account) {
+                        authentication = subtask.open_account;
+                        break;
+                    } else if (subtask.subtask_id === 'LoginTwoFactorAuthChallenge') {
+                        const token = authenticator.generate(authenticationSecret);
+
+                        const task5 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
+                            headers,
+                            json: {
+                                flow_token: task4.data.flow_token,
+                                subtask_inputs: [
+                                    {
+                                        enter_text: {
+                                            suggestion_id: null,
+                                            text: token,
+                                            link: 'next_link',
+                                        },
+                                        subtask_id: 'LoginTwoFactorAuthChallenge',
                                     },
-                                    subtask_id: 'LoginTwoFactorAuthChallenge',
-                                },
-                            ],
-                        },
-                    });
-                    logger.debug('Twitter login 6 finished: LoginTwoFactorAuthChallenge.');
-
-                    for (const subtask of task5.data?.subtasks || []) {
-                        if (subtask.open_account) {
-                            authentication = subtask.open_account;
-                            break;
+                                ],
+                            },
+                        });
+                        logger.debug('Twitter login 6 finished: LoginTwoFactorAuthChallenge.');
+
+                        for (const subtask of task5.data?.subtasks || []) {
+                            if (subtask.open_account) {
+                                authentication = subtask.open_account;
+                                break;
+                            }
                         }
+                        break;
+                    } else {
+                        logger.error(`Twitter login 6 failed: unknown subtask: ${subtask.subtask_id}`);
                     }
-                    break;
                 }
-            }
-            if (authentication) {
-                logger.debug('Twitter login success.');
-            } else {
-                logger.error(`Twitter login failed. ${JSON.stringify(task4.data?.subtasks, null, 2)}`);
-            }
+                if (authentication) {
+                    logger.debug('Twitter login success.');
+                } else {
+                    logger.error(`Twitter login failed. ${JSON.stringify(task4.data?.subtasks, null, 2)}`);
+                }
 
-            return authentication;
+                return authentication;
+            } catch (error) {
+                logger.error(`Twitter username ${username} login failed:`, error);
+            }
         },
         60 * 60 * 24 * 30, // 30 days
         false
-    );
+    )) as {
+        oauth_token: string;
+        oauth_token_secret: string;
+    } | null;
 }
 
 export default login;

package/lib/routes/twitter/api/mobile-api/token.ts

@@ -3,35 +3,29 @@ import login from './login';
 import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 let tokenIndex = 0;
-let authentication = null;
-let first = true;
 
-async function initToken() {
-    if (config.twitter.username && config.twitter.password && !authentication && first) {
-        authentication = await login();
-        first = false;
-    }
-}
-
-function getToken() {
+async function getToken() {
     let token;
     if (config.twitter.username && config.twitter.password) {
-        if (authentication) {
+        const index = tokenIndex++ % config.twitter.username.length;
+        const username = config.twitter.username[index];
+        const password = config.twitter.password[index];
+        const authenticationSecret = config.twitter.authenticationSecret?.[index];
+        if (username && password) {
+            const authentication = await login({
+                username,
+                password,
+                authenticationSecret,
+            });
+            if (!authentication) {
+                throw new ConfigNotFoundError(`Invalid twitter configs: ${username}`);
+            }
             token = {
                 key: authentication.oauth_token,
                 secret: authentication.oauth_token_secret,
+                cacheKey: `twitter:authentication:${username}`,
             };
         }
-    } else if (config.twitter.oauthTokens?.length && config.twitter.oauthTokenSecrets.length && config.twitter.oauthTokens.length === config.twitter.oauthTokenSecrets.length) {
-        token = {
-            key: config.twitter.oauthTokens[tokenIndex],
-            secret: config.twitter.oauthTokenSecrets[tokenIndex],
-        };
-
-        tokenIndex++;
-        if (tokenIndex >= config.twitter.oauthTokens.length) {
-            tokenIndex = 0;
-        }
     } else {
         throw new ConfigNotFoundError('Invalid twitter configs');
     }
@@ -39,4 +33,4 @@ function getToken() {
     return token;
 }
 
-export { initToken, getToken };
+export { getToken };

package/lib/routes/twitter/api/web-api/api.ts

@@ -34,6 +34,7 @@ const cacheTryGet = async (_id, params, func) => {
     const userData: any = await getUserData(_id);
     const id = (userData.data?.user || userData.data?.user_result)?.result?.rest_id;
     if (id === undefined) {
+        cache.set(`twitter-userdata-${_id}`, '', config.cache.contentExpire);
         throw new InvalidParameterError('User not found');
     }
     const funcName = func.name;

package/lib/routes/twitter/api/web-api/utils.ts

@@ -3,25 +3,89 @@ import { baseUrl, gqlFeatures, bearerToken, gqlMap } from './constants';
 import { config } from '@/config';
 import got from '@/utils/got';
 import queryString from 'query-string';
-import { Cookie } from 'tough-cookie';
+import { Cookie, CookieJar } from 'tough-cookie';
+import { CookieAgent, CookieClient } from 'http-cookie-agent/undici';
+import { ProxyAgent } from 'undici';
+import cache from '@/utils/cache';
+import logger from '@/utils/logger';
+import { RateLimiterMemory, RateLimiterRedis, RateLimiterQueue } from 'rate-limiter-flexible';
+import ofetch from '@/utils/ofetch';
+import proxy from '@/utils/proxy';
+
+const dispatchers = {};
+let authTokenIndex = 0;
+
+const loginLimiter = cache.clients.redisClient
+    ? new RateLimiterRedis({
+          points: 1,
+          duration: 5,
+          execEvenly: true,
+          storeClient: cache.clients.redisClient,
+      })
+    : new RateLimiterMemory({
+          points: 1,
+          duration: 5,
+          execEvenly: true,
+      });
+
+const loginLimiterQueue = new RateLimiterQueue(loginLimiter);
+
+const token2Cookie = (token) =>
+    cache.tryGet(`twitter:cookie:${token}`, async () => {
+        const jar = new CookieJar();
+        jar.setCookieSync(`auth_token=${token}`, 'https://x.com');
+        try {
+            const agent = proxy.proxyUri
+                ? new ProxyAgent({
+                      factory: (origin, opts) => new CookieClient(origin as string, { ...opts, cookies: { jar } }),
+                      uri: proxy.proxyUri,
+                  })
+                : new CookieAgent({ cookies: { jar } });
+            await got('https://x.com', {
+                dispatcher: agent,
+            });
+            return JSON.stringify(jar.serializeSync());
+        } catch {
+            // ignore
+            return '';
+        }
+    });
 
 export const twitterGot = async (url, params) => {
-    if (!config.twitter.cookie) {
+    if (!config.twitter.authToken) {
         throw new ConfigNotFoundError('Twitter cookie is not configured');
     }
+    await loginLimiterQueue.removeTokens(1);
+    const token = config.twitter.authToken[authTokenIndex++ % config.twitter.authToken.length];
+    let cookie = await token2Cookie(token);
+    if (cookie) {
+        logger.debug(`Got twitter cookie for token ${token}`);
+        if (typeof cookie === 'string') {
+            cookie = JSON.parse(cookie);
+        }
+        const jar = CookieJar.deserializeSync(cookie as any);
+        const agent = proxy.proxyUri
+            ? new ProxyAgent({
+                  factory: (origin, opts) => new CookieClient(origin as string, { ...opts, cookies: { jar } }),
+                  uri: proxy.proxyUri,
+              })
+            : new CookieAgent({ cookies: { jar } });
+        dispatchers[token] = {
+            jar,
+            agent,
+        };
+    } else {
+        throw new ConfigNotFoundError(`Twitter cookie for token ${token} is not valid`);
+    }
     const jsonCookie = Object.fromEntries(
-        config.twitter.cookie
+        dispatchers[token].jar
+            .getCookieStringSync(url)
             .split(';')
             .map((c) => Cookie.parse(c)?.toJSON())
             .map((c) => [c?.key, c?.value])
     );
-    if (!jsonCookie || !jsonCookie.auth_token || !jsonCookie.ct0) {
-        throw new ConfigNotFoundError('Twitter cookie is not valid');
-    }
 
-    const requestData = {
-        url: `${url}?${queryString.stringify(params)}`,
-        method: 'GET',
+    const response = await ofetch.raw(`${url}?${queryString.stringify(params)}`, {
         headers: {
             authority: 'x.com',
             accept: '*/*',
@@ -29,22 +93,29 @@ export const twitterGot = async (url, params) => {
             authorization: bearerToken,
             'cache-control': 'no-cache',
             'content-type': 'application/json',
-            cookie: config.twitter.cookie,
             dnt: '1',
             pragma: 'no-cache',
             referer: 'https://x.com/narendramodi',
-            'x-csrf-token': jsonCookie.ct0,
             'x-twitter-active-user': 'yes',
             'x-twitter-auth-type': 'OAuth2Session',
             'x-twitter-client-language': 'en',
+            'x-csrf-token': jsonCookie.ct0,
+        },
+        dispatcher: dispatchers[token].agent,
+        onResponse: async ({ response }) => {
+            if (response.status === 403) {
+                logger.debug(`Delete twitter cookie for token ${token}`);
+                await cache.set(`twitter:cookie:${token}`, '', config.cache.contentExpire);
+            }
         },
-    };
-
-    const response = await got(requestData.url, {
-        headers: requestData.headers,
     });
 
-    return response.data;
+    if (token) {
+        logger.debug(`Reset twitter cookie for token ${token}`);
+        await cache.set(`twitter:cookie:${token}`, JSON.stringify(dispatchers[token].jar.serializeSync()), config.cache.contentExpire);
+    }
+
+    return response._data;
 };
 
 export const paginationTweets = async (endpoint: string, userId: number | undefined, variables: Record<string, any>, path?: string[]) => {
@@ -66,7 +137,8 @@ export const paginationTweets = async (endpoint: string, userId: number | undefi
         if (data?.user?.result?.timeline_v2?.timeline?.instructions) {
             instructions = data.user.result.timeline_v2.timeline.instructions;
         } else {
-            throw new Error('Because Twitter Premium has features that hide your likes, this RSS link is not available for Twitter Premium accounts.');
+            // throw new Error('Because Twitter Premium has features that hide your likes, this RSS link is not available for Twitter Premium accounts.');
+            logger.debug(`Instructions not found in data: ${JSON.stringify(data)}`);
         }
     }
 

package/lib/routes/twitter/home.ts

@@ -17,7 +17,7 @@ export const route: Route = {
                 description: 'Please see above for details.',
             },
             {
-                name: 'TWITTER_COOKIE',
+                name: 'TWITTER_AUTH_TOKEN',
                 description: 'Please see above for details.',
             },
         ],
@@ -28,7 +28,7 @@ export const route: Route = {
         supportScihub: false,
     },
     name: 'Home timeline',
-    maintainers: ['DIYgod'],
+    maintainers: ['DIYgod', 'CaoMeiYouRen'],
     handler,
     radar: [
         {

package/lib/routes/twitter/keyword.ts

@@ -1,10 +1,11 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import api from './api';
 import utils from './utils';
 
 export const route: Route = {
     path: '/keyword/:keyword/:routeParams?',
     categories: ['social-media', 'popular'],
+    view: ViewType.SocialMedia,
     example: '/twitter/keyword/RSSHub',
     parameters: { keyword: 'keyword', routeParams: 'extra parameters, see the table above' },
     features: {
@@ -18,7 +19,7 @@ export const route: Route = {
                 description: 'Please see above for details.',
             },
             {
-                name: 'TWITTER_COOKIE',
+                name: 'TWITTER_AUTH_TOKEN',
                 description: 'Please see above for details.',
             },
         ],

package/lib/routes/twitter/likes.ts

@@ -10,7 +10,7 @@ export const route: Route = {
     features: {
         requireConfig: [
             {
-                name: 'TWITTER_COOKIE',
+                name: 'TWITTER_AUTH_TOKEN',
                 description: 'Please see above for details.',
             },
         ],

package/lib/routes/twitter/list.ts

@@ -18,7 +18,7 @@ export const route: Route = {
                 description: 'Please see above for details.',
             },
             {
-                name: 'TWITTER_COOKIE',
+                name: 'TWITTER_AUTH_TOKEN',
                 description: 'Please see above for details.',
             },
         ],

package/lib/routes/twitter/media.ts

@@ -1,11 +1,12 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import api from './api';
 import utils from './utils';
 
 export const route: Route = {
     path: '/media/:id/:routeParams?',
     categories: ['social-media', 'popular'],
-    example: '/twitter/media/DIYgod',
+    view: ViewType.Pictures,
+    example: '/twitter/media/_RSSHub',
     parameters: { id: 'username; in particular, if starts with `+`, it will be recognized as a [unique ID](https://github.com/DIYgod/RSSHub/issues/12221), e.g. `+44196397`', routeParams: 'extra parameters, see the table above.' },
     features: {
         requireConfig: [
@@ -18,7 +19,7 @@ export const route: Route = {
                 description: 'Please see above for details.',
             },
             {
-                name: 'TWITTER_COOKIE',
+                name: 'TWITTER_AUTH_TOKEN',
                 description: 'Please see above for details.',
             },
         ],

package/lib/routes/twitter/namespace.ts

@@ -40,8 +40,8 @@ generates
 
 Currently supports two authentication methods:
 
-- Using \`TWITTER_COOKIE\` (recommended): Configure the cookies of logged-in Twitter Web, at least including the fields auth_token and ct0. RSSHub will use this information to directly access Twitter's web API to obtain data.
+- Using \`TWITTER_AUTH_TOKEN\` (recommended): Configure a comma-separated list of \`auth_token\` cookies of logged-in Twitter Web. RSSHub will use this information to directly access Twitter's web API to obtain data.
 
-- Using \`TWITTER_USERNAME\` \`TWITTER_PASSWORD\` and \`TWITTER_AUTHENTICATION_SECRET\`: Configure the Twitter username and password. RSSHub will use this information to log in to Twitter and obtain data using the mobile API. Please note that if you have not logged in with the current IP address before, it is easy to trigger Twitter's risk control mechanism.
+- Using \`TWITTER_USERNAME\` \`TWITTER_PASSWORD\` and \`TWITTER_AUTHENTICATION_SECRET\`: Configure a comma-separated list of Twitter username and password. RSSHub will use this information to log in to Twitter and obtain data using the mobile API. Please note that if you have not logged in with the current IP address before, it is easy to trigger Twitter's risk control mechanism.
 `,
 };