rsshub 1.0.0-master.f71451d → 1.0.0-master.f72af1b

package/lib/routes/ttv/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '台視新聞網',
+    url: 'news.ttv.com.tw',
+};

package/lib/routes/tvb/news.ts

@@ -96,6 +96,7 @@ async function handler(ctx) {
             lang: language,
             page: 1,
             limit: ctx.req.query('limit') ?? 50,
+            country: 'HK',
         },
     });
 

package/lib/routes/twitch/live.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
 
@@ -7,7 +7,8 @@ const TWITCH_CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko';
 
 export const route: Route = {
     path: '/live/:login',
-    categories: ['live'],
+    categories: ['live', 'popular'],
+    view: ViewType.Notifications,
     example: '/twitch/live/riotgames',
     parameters: { login: 'Twitch username' },
     features: {

package/lib/routes/twitch/video.ts

@@ -1,5 +1,5 @@
 import InvalidParameterError from '@/errors/types/invalid-parameter';
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import got from '@/utils/got';
 import { parseDate } from '@/utils/parse-date';
 
@@ -14,9 +14,21 @@ const FILTER_NODE_TYPE_MAP = {
 
 export const route: Route = {
     path: '/video/:login/:filter?',
-    categories: ['live'],
+    categories: ['live', 'popular'],
+    view: ViewType.Videos,
     example: '/twitch/video/riotgames/highlights',
-    parameters: { login: 'Twitch username', filter: 'Video type, Default to all' },
+    parameters: {
+        login: 'Twitch username',
+        filter: {
+            description: 'Video type, Default to all',
+            options: [
+                { value: 'archive', label: 'Archive' },
+                { value: 'highlights', label: 'Highlights' },
+                { value: 'all', label: 'All' },
+            ],
+            default: 'all',
+        },
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,
@@ -34,9 +46,6 @@ export const route: Route = {
     name: 'Channel Video',
     maintainers: ['hoilc'],
     handler,
-    description: `| archive           | highlights                    | all        |
-| ----------------- | ----------------------------- | ---------- |
-| Recent broadcasts | Recent highlights and uploads | All videos |`,
 };
 
 async function handler(ctx) {

package/lib/routes/twitter/api/developer-api/search.ts

@@ -13,7 +13,7 @@ const handler = async (ctx) => {
 
     return {
         title: `Twitter Keyword - ${keyword}`,
-        link: `https://twitter.com/search?q=${encodeURIComponent(keyword)}`,
+        link: `https://x.com/search?q=${encodeURIComponent(keyword)}`,
         item: utils.ProcessFeed(ctx, {
             data: data.statuses,
         }),

package/lib/routes/twitter/api/developer-api/user.ts

@@ -27,7 +27,7 @@ const handler = async (ctx) => {
 
     return {
         title: `Twitter @${userInfo.name}`,
-        link: `https://twitter.com/${screen_name}`,
+        link: `https://x.com/${screen_name}`,
         image: profileImageUrl,
         description: userInfo.description,
         item: utils.ProcessFeed(ctx, {

package/lib/routes/twitter/api/index.ts

@@ -4,7 +4,7 @@ import webApi from './web-api/api';
 import { config } from '@/config';
 
 const enableMobileApi = config.twitter.username && config.twitter.password;
-const enableWebApi = config.twitter.cookie;
+const enableWebApi = config.twitter.authToken;
 
 type ApiItem = (id: string, params?: Record<string, any>) => Promise<Record<string, any>> | Record<string, any> | null;
 let api: {
@@ -18,6 +18,7 @@ let api: {
     getSearch: ApiItem;
     getList: ApiItem;
     getHomeTimeline: ApiItem;
+    getHomeLatestTimeline: ApiItem;
 } = {
     init: () => {
         throw new ConfigNotFoundError('Twitter API is not configured');
@@ -31,6 +32,7 @@ let api: {
     getSearch: () => null,
     getList: () => null,
     getHomeTimeline: () => null,
+    getHomeLatestTimeline: () => null,
 };
 
 if (enableWebApi) {

package/lib/routes/twitter/api/mobile-api/api.ts

@@ -1,18 +1,18 @@
 import { baseUrl, gqlMap, gqlFeatures, consumerKey, consumerSecret } from './constants';
 import { config } from '@/config';
 import logger from '@/utils/logger';
-import got from '@/utils/got';
 import OAuth from 'oauth-1.0a';
 import CryptoJS from 'crypto-js';
 import queryString from 'query-string';
-import { initToken, getToken } from './token';
+import { getToken } from './token';
 import cache from '@/utils/cache';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
+import ofetch from '@/utils/ofetch';
 
 const twitterGot = async (url, params) => {
     const token = await getToken();
 
-    const oauth = OAuth({
+    const oauth = new OAuth({
         consumer: {
             key: consumerKey,
             secret: consumerSecret,
@@ -28,7 +28,7 @@ const twitterGot = async (url, params) => {
             connection: 'keep-alive',
             'content-type': 'application/json',
             'x-twitter-active-user': 'yes',
-            authority: 'api.twitter.com',
+            authority: 'api.x.com',
             'accept-encoding': 'gzip',
             'accept-language': 'en-US,en;q=0.9',
             accept: '*/*',
@@ -36,11 +36,14 @@ const twitterGot = async (url, params) => {
         },
     };
 
-    const response = await got(requestData.url, {
+    const response = await ofetch.raw(requestData.url, {
         headers: oauth.toHeader(oauth.authorize(requestData, token)),
     });
+    if (response.status === 401) {
+        cache.globalCache.set(token.cacheKey, '');
+    }
 
-    return response.data;
+    return response._data;
 };
 
 const paginationTweets = async (endpoint, userId, variables, path) => {
@@ -279,5 +282,5 @@ export default {
     excludeRetweet,
     getSearch,
     getUserTweet,
-    init: initToken,
+    init: () => void 0,
 };

package/lib/routes/twitter/api/mobile-api/constants.ts

@@ -1,4 +1,4 @@
-const baseUrl = 'https://api.twitter.com';
+const baseUrl = 'https://api.x.com';
 
 const consumerKey = '3nVuSoBZnx6U4vzUxf5w';
 const consumerSecret = 'Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys';

package/lib/routes/twitter/api/mobile-api/login.ts

@@ -4,14 +4,13 @@ import { bearerToken, guestActivateUrl } from './constants';
 import got from '@/utils/got';
 import ofetch from '@/utils/ofetch';
 import crypto from 'crypto';
-import { config } from '@/config';
 import { v5 as uuidv5 } from 'uuid';
+import { authenticator } from 'otplib';
 import logger from '@/utils/logger';
 import cache from '@/utils/cache';
+import { RateLimiterMemory, RateLimiterRedis, RateLimiterQueue } from 'rate-limiter-flexible';
 
 const NAMESPACE = 'd41d092b-b007-48f7-9129-e9538d2d8fe9';
-const username = config.twitter.username;
-const password = config.twitter.password;
 
 let authentication = null;
 
@@ -27,158 +26,183 @@ const headers = {
     Authorization: bearerToken,
 };
 
-async function login() {
-    return await cache.tryGet(
+const loginLimiter = cache.clients.redisClient
+    ? new RateLimiterRedis({
+          points: 1,
+          duration: 20,
+          execEvenly: true,
+          storeClient: cache.clients.redisClient,
+      })
+    : new RateLimiterMemory({
+          points: 1,
+          duration: 20,
+          execEvenly: true,
+      });
+
+const loginLimiterQueue = new RateLimiterQueue(loginLimiter);
+
+async function login({ username, password, authenticationSecret }) {
+    return (await cache.tryGet(
         `twitter:authentication:${username}`,
         async () => {
-            logger.debug('Twitter login start.');
-            const android_id = uuidv5(username, NAMESPACE);
-            headers['X-Twitter-Client-DeviceID'] = android_id;
-
-            const ct0 = crypto.randomUUID().replaceAll('-', '');
-            const guestToken = await got(guestActivateUrl, {
-                headers: {
-                    authorization: bearerToken,
-                    'x-csrf-token': ct0,
-                    cookie: 'ct0=' + ct0,
-                },
-                method: 'POST',
-            });
-            logger.debug('Twitter login 1 finished: guest token.');
-
-            headers['x-guest-token'] = guestToken.data.guest_token;
-
-            const task1 = await ofetch.raw(
-                'https://api.twitter.com/1.1/onboarding/task.json?' +
-                    new URLSearchParams({
-                        flow_name: 'login',
-                        api_version: '1',
-                        known_device_token: '',
-                        sim_country_code: 'us',
-                    }).toString(),
-                {
+            try {
+                await loginLimiterQueue.removeTokens(1);
+
+                logger.debug('Twitter login start.');
+                const android_id = uuidv5(username, NAMESPACE);
+                headers['X-Twitter-Client-DeviceID'] = android_id;
+
+                const ct0 = crypto.randomUUID().replaceAll('-', '');
+                const guestToken = await got(guestActivateUrl, {
+                    headers: {
+                        authorization: bearerToken,
+                        'x-csrf-token': ct0,
+                        cookie: 'ct0=' + ct0,
+                    },
                     method: 'POST',
-                    headers,
-                    body: {
-                        flow_token: null,
-                        input_flow_data: {
-                            country_code: null,
-                            flow_context: {
-                                referrer_context: {
-                                    referral_details: 'utm_source=google-play&utm_medium=organic',
-                                    referrer_url: '',
-                                },
-                                start_location: {
-                                    location: 'deeplink',
+                });
+                logger.debug('Twitter login 1 finished: guest token.');
+
+                headers['x-guest-token'] = guestToken.data.guest_token;
+
+                const task1 = await ofetch.raw(
+                    'https://api.x.com/1.1/onboarding/task.json?' +
+                        new URLSearchParams({
+                            flow_name: 'login',
+                            api_version: '1',
+                            known_device_token: '',
+                            sim_country_code: 'us',
+                        }).toString(),
+                    {
+                        method: 'POST',
+                        headers,
+                        body: {
+                            flow_token: null,
+                            input_flow_data: {
+                                country_code: null,
+                                flow_context: {
+                                    referrer_context: {
+                                        referral_details: 'utm_source=google-play&utm_medium=organic',
+                                        referrer_url: '',
+                                    },
+                                    start_location: {
+                                        location: 'deeplink',
+                                    },
                                 },
+                                requested_variant: null,
+                                target_user_id: 0,
                             },
-                            requested_variant: null,
-                            target_user_id: 0,
                         },
+                    }
+                );
+                logger.debug('Twitter login 2 finished: login flow.');
+
+                headers.att = task1.headers.get('att');
+
+                const task2 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
+                    headers,
+                    json: {
+                        flow_token: task1._data.flow_token,
+                        subtask_inputs: [
+                            {
+                                enter_text: {
+                                    suggestion_id: null,
+                                    text: username,
+                                    link: 'next_link',
+                                },
+                                subtask_id: 'LoginEnterUserIdentifier',
+                            },
+                        ],
                     },
-                }
-            );
-            logger.debug('Twitter login 2 finished: login flow.');
-
-            headers.att = task1.headers.get('att');
-
-            const task2 = await got.post('https://api.twitter.com/1.1/onboarding/task.json', {
-                headers,
-                json: {
-                    flow_token: task1._data.flow_token,
-                    subtask_inputs: [
-                        {
-                            enter_text: {
-                                suggestion_id: null,
-                                text: username,
-                                link: 'next_link',
+                });
+                logger.debug('Twitter login 3 finished: LoginEnterUserIdentifier.');
+
+                const task3 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
+                    headers,
+                    json: {
+                        flow_token: task2.data.flow_token,
+                        subtask_inputs: [
+                            {
+                                enter_password: {
+                                    password,
+                                    link: 'next_link',
+                                },
+                                subtask_id: 'LoginEnterPassword',
                             },
-                            subtask_id: 'LoginEnterUserIdentifier',
-                        },
-                    ],
-                },
-            });
-            logger.debug('Twitter login 3 finished: LoginEnterUserIdentifier.');
-
-            const task3 = await got.post('https://api.twitter.com/1.1/onboarding/task.json', {
-                headers,
-                json: {
-                    flow_token: task2.data.flow_token,
-                    subtask_inputs: [
-                        {
-                            enter_password: {
-                                password,
-                                link: 'next_link',
+                        ],
+                    },
+                });
+                logger.debug('Twitter login 4 finished: LoginEnterPassword.');
+
+                const task4 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
+                    headers,
+                    json: {
+                        flow_token: task3.data.flow_token,
+                        subtask_inputs: [
+                            {
+                                check_logged_in_account: {
+                                    link: 'AccountDuplicationCheck_false',
+                                },
+                                subtask_id: 'AccountDuplicationCheck',
                             },
-                            subtask_id: 'LoginEnterPassword',
-                        },
-                    ],
-                },
-            });
-            logger.debug('Twitter login 4 finished: LoginEnterPassword.');
-
-            const task4 = await got.post('https://api.twitter.com/1.1/onboarding/task.json', {
-                headers,
-                json: {
-                    flow_token: task3.data.flow_token,
-                    subtask_inputs: [
-                        {
-                            check_logged_in_account: {
-                                link: 'AccountDuplicationCheck_false',
+                        ],
+                    },
+                });
+                logger.debug('Twitter login 5 finished: AccountDuplicationCheck.');
+
+                for await (const subtask of task4.data?.subtasks || []) {
+                    if (subtask.open_account) {
+                        authentication = subtask.open_account;
+                        break;
+                    } else if (authenticationSecret && subtask.subtask_id === 'LoginTwoFactorAuthChallenge') {
+                        const token = authenticator.generate(authenticationSecret);
+
+                        const task5 = await got.post('https://api.x.com/1.1/onboarding/task.json', {
+                            headers,
+                            json: {
+                                flow_token: task4.data.flow_token,
+                                subtask_inputs: [
+                                    {
+                                        enter_text: {
+                                            suggestion_id: null,
+                                            text: token,
+                                            link: 'next_link',
+                                        },
+                                        subtask_id: 'LoginTwoFactorAuthChallenge',
+                                    },
+                                ],
                             },
-                            subtask_id: 'AccountDuplicationCheck',
-                        },
-                    ],
-                },
-            });
-            logger.debug('Twitter login 5 finished: AccountDuplicationCheck.');
-
-            for (const subtask of task4.data?.subtasks || []) {
-                if (subtask.open_account) {
-                    authentication = subtask.open_account;
-                    break;
-                } else if (subtask.subtask_id === 'LoginTwoFactorAuthChallenge') {
-                    // const token = authenticator.generate(authenticationSecret);
-
-                    // // eslint-disable-next-line no-await-in-loop
-                    // const task5 = await got.post('https://api.twitter.com/1.1/onboarding/task.json', {
-                    //     headers,
-                    //     json: {
-                    //         flow_token: task4.data.flow_token,
-                    //         subtask_inputs: [
-                    //             {
-                    //                 enter_text: {
-                    //                     suggestion_id: null,
-                    //                     text: token,
-                    //                     link: 'next_link',
-                    //                 },
-                    //                 subtask_id: 'LoginTwoFactorAuthChallenge',
-                    //             },
-                    //         ],
-                    //     },
-                    // });
-                    // logger.debug('Twitter login 6 finished: LoginTwoFactorAuthChallenge.');
-
-                    // for (const subtask of task5.data?.subtasks || []) {
-                    //     if (subtask.open_account) {
-                    //         authentication = subtask.open_account;
-                    //         break;
-                    //     }
-                    // }
-                    break;
+                        });
+                        logger.debug('Twitter login 6 finished: LoginTwoFactorAuthChallenge.');
+
+                        for (const subtask of task5.data?.subtasks || []) {
+                            if (subtask.open_account) {
+                                authentication = subtask.open_account;
+                                break;
+                            }
+                        }
+                        break;
+                    } else {
+                        logger.error(`Twitter login 6 failed: unknown subtask: ${subtask.subtask_id}`);
+                    }
+                }
+                if (authentication) {
+                    logger.debug('Twitter login success.');
+                } else {
+                    logger.error(`Twitter login failed. ${JSON.stringify(task4.data?.subtasks, null, 2)}`);
                 }
-            }
-            if (authentication) {
-                logger.debug('Twitter login success.');
-            } else {
-                logger.error(`Twitter login failed. ${JSON.stringify(task4.data?.subtasks, null, 2)}`);
-            }
 
-            return authentication;
+                return authentication;
+            } catch (error) {
+                logger.error(`Twitter username ${username} login failed:`, error);
+            }
         },
         60 * 60 * 24 * 30, // 30 days
         false
-    );
+    )) as {
+        oauth_token: string;
+        oauth_token_secret: string;
+    } | null;
 }
 
 export default login;

package/lib/routes/twitter/api/mobile-api/token.ts

@@ -3,35 +3,29 @@ import login from './login';
 import ConfigNotFoundError from '@/errors/types/config-not-found';
 
 let tokenIndex = 0;
-let authentication = null;
-let first = true;
 
-async function initToken() {
-    if (config.twitter.username && config.twitter.password && !authentication && first) {
-        authentication = await login();
-        first = false;
-    }
-}
-
-function getToken() {
+async function getToken() {
     let token;
     if (config.twitter.username && config.twitter.password) {
-        if (authentication) {
+        const index = tokenIndex++ % config.twitter.username.length;
+        const username = config.twitter.username[index];
+        const password = config.twitter.password[index];
+        const authenticationSecret = config.twitter.authenticationSecret?.[index];
+        if (username && password) {
+            const authentication = await login({
+                username,
+                password,
+                authenticationSecret,
+            });
+            if (!authentication) {
+                throw new ConfigNotFoundError(`Invalid twitter configs: ${username}`);
+            }
             token = {
                 key: authentication.oauth_token,
                 secret: authentication.oauth_token_secret,
+                cacheKey: `twitter:authentication:${username}`,
             };
         }
-    } else if (config.twitter.oauthTokens?.length && config.twitter.oauthTokenSecrets.length && config.twitter.oauthTokens.length === config.twitter.oauthTokenSecrets.length) {
-        token = {
-            key: config.twitter.oauthTokens[tokenIndex],
-            secret: config.twitter.oauthTokenSecrets[tokenIndex],
-        };
-
-        tokenIndex++;
-        if (tokenIndex >= config.twitter.oauthTokens.length) {
-            tokenIndex = 0;
-        }
     } else {
         throw new ConfigNotFoundError('Invalid twitter configs');
     }
@@ -39,4 +33,4 @@ function getToken() {
     return token;
 }
 
-export { initToken, getToken };
+export { getToken };

package/lib/routes/twitter/api/web-api/api.ts

@@ -34,6 +34,7 @@ const cacheTryGet = async (_id, params, func) => {
     const userData: any = await getUserData(_id);
     const id = (userData.data?.user || userData.data?.user_result)?.result?.rest_id;
     if (id === undefined) {
+        cache.set(`twitter-userdata-${_id}`, '', config.cache.contentExpire);
         throw new InvalidParameterError('User not found');
     }
     const funcName = func.name;
@@ -97,7 +98,19 @@ const getUserMedia = (id: string, params?: Record<string, any>) =>
         );
     });
 
-const getUserLikes = (id: string, params?: Record<string, any>) => cacheTryGet(id, params, async (id, params = {}) => gatherLegacyFromData(await paginationTweets('Likes', id, params)));
+const getUserLikes = (id: string, params?: Record<string, any>) =>
+    cacheTryGet(id, params, async (id, params = {}) =>
+        gatherLegacyFromData(
+            await paginationTweets('Likes', id, {
+                ...params,
+                includeHasBirdwatchNotes: false,
+                includePromotedContent: false,
+                withBirdwatchNotes: false,
+                withVoice: false,
+                withV2Timeline: true,
+            })
+        )
+    );
 
 const getUserTweet = (id: string, params?: Record<string, any>) =>
     cacheTryGet(id, params, async (id, params = {}) =>
@@ -171,6 +184,23 @@ const getHomeTimeline = async (id: string, params?: Record<string, any>) =>
         )
     );
 
+const getHomeLatestTimeline = async (id: string, params?: Record<string, any>) =>
+    gatherLegacyFromData(
+        await paginationTweets(
+            'HomeLatestTimeline',
+            undefined,
+            {
+                ...params,
+                count: 20,
+                includePromotedContent: true,
+                latestControlAvailable: true,
+                requestContext: 'launch',
+                withCommunity: true,
+            },
+            ['home', 'home_timeline_urt']
+        )
+    );
+
 export default {
     getUser,
     getUserTweets,
@@ -181,5 +211,6 @@ export default {
     getSearch,
     getList,
     getHomeTimeline,
+    getHomeLatestTimeline,
     init: () => {},
 };