rsshub 1.0.0-master.fcb92f0 → 1.0.0-master.fcc0fab

package/lib/v2/tradingview/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: 'Blog',
-                docs: 'https://docs.rsshub.app/program-update.html#tradingview-blog',
+                docs: 'https://docs.rsshub.app/routes/program-update#tradingview-blog',
                 source: ['/blog/:language', '/'],
                 target: '/tradingview/blog',
             },

package/lib/v2/trending/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         so: [
             {
                 title: '热搜关键词聚合追踪',
-                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                docs: 'https://docs.rsshub.app/routes/social-media#re-sou-ju-he',
                 source: ['/search'],
                 target: (params, url) => `/trending/${new URL(url).searchParams.get('keyword')}`,
             },
@@ -15,7 +15,7 @@ module.exports = {
         s: [
             {
                 title: '热搜关键词聚合追踪',
-                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                docs: 'https://docs.rsshub.app/routes/social-media#re-sou-ju-he',
                 source: '/weibo/:keyword',
                 target: (params) => `/trending/${params.keyword}}`,
             },
@@ -26,7 +26,7 @@ module.exports = {
         www: [
             {
                 title: '热搜关键词聚合追踪',
-                docs: 'https://docs.rsshub.app/social-media.html#re-sou-ju-he',
+                docs: 'https://docs.rsshub.app/routes/social-media#re-sou-ju-he',
                 source: ['/search'],
                 target: (params, url) => `/trending/${new URL(url).searchParams.get('q')}`,
             },

package/lib/v2/tribalfootball/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: 'Latest News',
-                docs: 'https://docs.rsshub.app/new-media.html#tribal-football',
+                docs: 'https://docs.rsshub.app/routes/new-media#tribal-football',
                 source: ['/'],
                 target: '/tribalfootball',
             },

package/lib/v2/trow/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '首页更新',
-                docs: 'https://docs.rsshub.app/bbs.html#the-ring-of-wonder',
+                docs: 'https://docs.rsshub.app/routes/bbs#the-ring-of-wonder',
                 source: ['/'],
                 target: '/portal',
             },

package/lib/v2/tvb/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '新闻',
-                docs: 'https://docs.rsshub.app/traditional-media.html#wu-xian-xin-wen-xin-wen',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#wu-xian-xin-wen-xin-wen',
                 source: ['/:language/:category', '/'],
                 target: '/tvb/news/:category?/:language?',
             },

package/lib/v2/twitter/developer-api/user.js

@@ -2,24 +2,32 @@ const utils = require('../utils');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
-
     // For compatibility
     const { exclude_replies, include_rts, count } = utils.parseRouteParams(ctx.params.routeParams);
     const client = await utils.getAppClient();
-    const data = await client.v1.get('statuses/user_timeline.json', {
-        screen_name: id,
+    const user_timeline_query = {
         tweet_mode: 'extended',
         exclude_replies,
         include_rts,
         count,
-    });
+    };
+    let screen_name;
+    if (id.startsWith('+')) {
+        user_timeline_query.user_id = +id.slice(1);
+    } else {
+        user_timeline_query.screen_name = id;
+        screen_name = id;
+    }
+    const data = await client.v1.get('statuses/user_timeline.json', user_timeline_query);
     const userInfo = data[0].user;
-
+    if (!screen_name) {
+        screen_name = userInfo.screen_name;
+    }
     const profileImageUrl = userInfo.profile_image_url || userInfo.profile_image_url_https;
 
     ctx.state.data = {
         title: `Twitter @${userInfo.name}`,
-        link: `https://twitter.com/${id}`,
+        link: `https://twitter.com/${screen_name}`,
         image: profileImageUrl,
         description: userInfo.description,
         item: utils.ProcessFeed(ctx, {

package/lib/v2/twitter/maintainer.js

@@ -6,6 +6,6 @@ module.exports = {
     '/list/:id/:name/:routeParams?': ['xyqfer'],
     '/media/:id/:routeParams?': ['yindaheng98', 'Rongronggg9'],
     '/trends/:woeid?': ['sakamossan'],
-    '/tweet/:id/status/:status/:original?': ['LarchLiu'],
+    '/tweet/:id/status/:status/:original?': ['LarchLiu', 'Rongronggg9'],
     '/user/:id/:routeParams?': ['DIYgod', 'yindaheng98', 'Rongronggg9'],
 };

package/lib/v2/twitter/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '用户时间线',
-                docs: 'https://docs.rsshub.app/social-media.html#twitter',
+                docs: 'https://docs.rsshub.app/routes/social-media#twitter',
                 source: '/:id',
                 target: (params) => {
                     if (params.id !== 'home' && params.id !== 'explore' && params.id !== 'notifications' && params.id !== 'messages' && params.id !== 'explore' && params.id !== 'search') {
@@ -14,7 +14,7 @@ module.exports = {
             },
             {
                 title: '用户媒体时间线',
-                docs: 'https://docs.rsshub.app/social-media.html#twitter',
+                docs: 'https://docs.rsshub.app/routes/social-media#twitter',
                 source: '/:id/media',
                 target: (params) => {
                     if (!['home', 'explore', 'notifications', 'messages', 'explore', 'search'].includes(params.id)) {
@@ -24,7 +24,7 @@ module.exports = {
             },
             {
                 title: '用户关注时间线',
-                docs: 'https://docs.rsshub.app/social-media.html#twitter',
+                docs: 'https://docs.rsshub.app/routes/social-media#twitter',
                 source: '/:id',
                 target: (params) => {
                     if (params.id !== 'home' && params.id !== 'explore' && params.id !== 'notifications' && params.id !== 'messages' && params.id !== 'explore' && params.id !== 'search') {
@@ -34,7 +34,7 @@ module.exports = {
             },
             {
                 title: '用户喜欢列表',
-                docs: 'https://docs.rsshub.app/social-media.html#twitter',
+                docs: 'https://docs.rsshub.app/routes/social-media#twitter',
                 source: '/:id',
                 target: (params) => {
                     if (params.id !== 'home' && params.id !== 'explore' && params.id !== 'notifications' && params.id !== 'messages' && params.id !== 'explore' && params.id !== 'search') {
@@ -44,7 +44,7 @@ module.exports = {
             },
             {
                 title: '列表时间线',
-                docs: 'https://docs.rsshub.app/social-media.html#twitter',
+                docs: 'https://docs.rsshub.app/routes/social-media#twitter',
                 source: '/:id/lists/:name',
                 target: (params) => {
                     if (params.id !== 'home' && params.id !== 'explore' && params.id !== 'notifications' && params.id !== 'messages' && params.id !== 'explore' && params.id !== 'search') {
@@ -54,13 +54,13 @@ module.exports = {
             },
             {
                 title: '关键词',
-                docs: 'https://docs.rsshub.app/social-media.html#twitter',
+                docs: 'https://docs.rsshub.app/routes/social-media#twitter',
                 source: '/search',
                 target: (params, url) => `/twitter/keyword/${new URL(url).searchParams.get('q')}`,
             },
             {
                 title: '推文收集',
-                docs: 'https://docs.rsshub.app/social-media.html#twitter',
+                docs: 'https://docs.rsshub.app/routes/social-media#twitter',
                 source: '/:uid/timelines/:collectionId',
                 target: (params) => {
                     if (params.uid !== 'home' && params.uid !== 'explore' && params.uid !== 'notifications' && params.uid !== 'messages' && params.uid !== 'explore' && params.uid !== 'search') {
@@ -70,7 +70,7 @@ module.exports = {
             },
             {
                 title: '推文详情',
-                docs: 'https://docs.rsshub.app/social-media.html#twitter',
+                docs: 'https://docs.rsshub.app/routes/social-media#twitter',
                 source: '/:id/status/:status',
                 target: (params) => {
                     if (params.id !== 'home' && params.id !== 'explore' && params.id !== 'notifications' && params.id !== 'messages' && params.id !== 'explore' && params.id !== 'search') {

package/lib/v2/twitter/utils.js

@@ -29,10 +29,16 @@ const getOriginalImg = (url) => {
 };
 const replaceBreak = (text) => text.replace(/<br><br>|<br>/g, ' ');
 
-const formatText = (text, urls) => {
+const formatText = (item) => {
+    let text = item.full_text;
+    const urls = item.entities.urls || [];
     urls.forEach((url) => {
         text = text.replaceAll(url.url, url.expanded_url);
     });
+    const media = item.extended_entities?.media || [];
+    media.forEach((m) => {
+        text = text.replaceAll(m.url, '');
+    });
     return text.trim().replace(/\n/g, '<br>');
 };
 
@@ -179,7 +185,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
         const originalItem = item;
         item = item.retweeted_status || item;
         item.full_text = item.full_text || item.text;
-        item.full_text = formatText(item.full_text, item.entities.urls);
+        item.full_text = formatText(item);
         const img = formatMedia(item);
         let picsPrefix = generatePicsPrefix(item);
         let quote = '';
@@ -223,7 +229,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
                 }
 
                 quote += `:&ensp;`;
-                quote += formatText(quoteData.full_text, quoteData.entities.urls);
+                quote += formatText(quoteData);
 
                 if (!readable) {
                     quote += '<br>';
@@ -231,7 +237,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
                 quote += formatMedia(quoteData);
                 picsPrefix += generatePicsPrefix(quoteData);
                 quoteInTitle += showEmojiForRetweetAndReply ? ' 💬 ' : showSymbolForRetweetAndReply ? ' RT ' : '';
-                quoteInTitle += `${author.name}: ${formatText(quoteData.full_text, quoteData.entities.urls)}`;
+                quoteInTitle += `${author.name}: ${formatText(quoteData)}`;
 
                 if (readable) {
                     quote += `<br><small>Link: <a href='https://twitter.com/${author.screen_name}/status/${quoteData.id_str}' target='_blank' rel='noopener noreferrer'>https://twitter.com/${author.screen_name}/status/${quoteData.id_str}</a></small>`;

package/lib/v2/twitter/web-api/constants.js

@@ -0,0 +1,77 @@
+// https://github.com/zedeus/nitter/issues/919#issuecomment-1619067142
+// https://git.sr.ht/~cloutier/bird.makeup/tree/4b2495bf2908c648ae9250f353bfdacf2464b98d/item/src/BirdsiteLive.Twitter/Tools/TwitterAuthenticationInitializer.cs#L36
+const auth = 'Bearer AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF';
+const tokens = [
+    auth, // tweetdeck new
+    // 'CjulERsDeqhhjSme66ECg:IQWdVyqFxghAtURHGeGiWAsmCAGmdW3WmbEx6Hck', // iPad, no mixed videos and photos support
+    // valid, but endpoints differ:
+    // 'IQKbtAYlXLripLGPWd0HUA:GgDYlkSvaPxGxC4X8liwpUoqKwwr3lCADbz8A7ADU', // iPhone
+    // '3nVuSoBZnx6U4vzUxf5w:Bcs59EFbbsdF6Sl9Ng71smgStWEGwXXKSjYvPVt7qys', // Android
+    // '3rJOl1ODzm9yZy63FACdg:5jPoQ5kQvMJFDYRNE8bQ4rHuds4xJqhvgNJM4awaE8', // Mac
+];
+
+const graphQLEndpointsPlain = [
+    '/graphql/oUZZZ8Oddwxs8Cd3iW3UEA/UserByScreenName',
+    '/graphql/Lxg1V9AiIzzXEiP2c8dRnw/UserByRestId',
+    '/graphql/3XDB26fBve-MmjHaWTUZxA/TweetDetail',
+    '/graphql/QqZBEqganhHwmU9QscmIug/UserTweets',
+    '/graphql/wxoVeDnl0mP7VLhe6mTOdg/UserTweetsAndReplies',
+    '/graphql/Az0-KW6F-FyYTc2OJmvUhg/UserMedia',
+    '/graphql/kgZtsNyE46T3JaEf2nF9vw/Likes',
+    // these endpoints are not available if authenticated as other clients
+    // FYI, endpoints for Android: https://gist.github.com/ScamCast/2e40befbd1b61c4a80cda2745d4df1f4
+];
+
+const graphQLMap = Object.fromEntries(graphQLEndpointsPlain.map((endpoint) => [endpoint.split('/')[3], endpoint]));
+
+// captured from Twitter web
+const featuresMap = {
+    UserByScreenName: JSON.stringify({
+        hidden_profile_likes_enabled: false,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        subscriptions_verification_info_verified_since_enabled: true,
+        highlights_tweets_tab_ui_enabled: true,
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+    }),
+    UserByRestId: JSON.stringify({
+        hidden_profile_likes_enabled: false,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        highlights_tweets_tab_ui_enabled: true,
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+    }),
+    UserTweets: JSON.stringify({
+        rweb_lists_timeline_redesign_enabled: true,
+        responsive_web_graphql_exclude_directive_enabled: true,
+        verified_phone_label_enabled: false,
+        creator_subscriptions_tweet_preview_api_enabled: true,
+        responsive_web_graphql_timeline_navigation_enabled: true,
+        responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+        tweetypie_unmention_optimization_enabled: true,
+        responsive_web_edit_tweet_api_enabled: true,
+        graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+        view_counts_everywhere_api_enabled: true,
+        longform_notetweets_consumption_enabled: true,
+        responsive_web_twitter_article_tweet_consumption_enabled: false,
+        tweet_awards_web_tipping_enabled: false,
+        freedom_of_speech_not_reach_fetch_enabled: true,
+        standardized_nudges_misinfo: true,
+        tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+        longform_notetweets_rich_text_read_enabled: true,
+        longform_notetweets_inline_media_enabled: true,
+        responsive_web_media_download_video_enabled: false,
+        responsive_web_enhance_cards_enabled: false,
+    }),
+};
+
+module.exports = {
+    auth,
+    tokens,
+    graphQLMap,
+    featuresMap,
+};

package/lib/v2/twitter/web-api/media.js

@@ -13,7 +13,7 @@ module.exports = async (ctx) => {
 
     ctx.state.data = {
         title: `Twitter @${userInfo.name}`,
-        link: `https://twitter.com/${id}/media`,
+        link: `https://twitter.com/${userInfo.screen_name}/media`,
         image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
         description: userInfo.description,
         item: utils.ProcessFeed(ctx, {

package/lib/v2/twitter/web-api/tweet.js

@@ -17,7 +17,7 @@ module.exports = async (ctx) => {
 
     ctx.state.data = {
         title: `Twitter @${userInfo.name}`,
-        link: `https://twitter.com/${id}/status/${status}`,
+        link: `https://twitter.com/${userInfo.screen_name}/status/${status}`,
         image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
         description: userInfo.description,
         item,

package/lib/v2/twitter/web-api/twitter-api.js

@@ -1,5 +1,8 @@
 const twitterGot = require('./twitter-got');
+const { graphQLMap, featuresMap, auth } = require('./constants');
 const config = require('@/config').value;
+const got = require('@/utils/got');
+const logger = require('@/utils/logger');
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L727-L755
 const _params = {
@@ -65,6 +68,7 @@ const paginationTweets = async (endpoint, userId, variables, path) => {
             ...variables,
             userId,
         }),
+        features: featuresMap.UserTweets,
     });
 
     let instructions;
@@ -81,40 +85,44 @@ const paginationTweets = async (endpoint, userId, variables, path) => {
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L807-L814
 const timelineTweets = (userId, params = {}) =>
-    paginationTweets('/graphql/WZT7sCTrLvSOaWOXLDsWbQ/UserTweets', userId, {
+    paginationTweets(graphQLMap.UserTweets, userId, {
         ...params,
         withQuickPromoteEligibilityTweetFields: true,
     });
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L816-L823
 const timelineTweetsAndReplies = (userId, params = {}) =>
-    paginationTweets('/graphql/t4wEKVulW4Mbv1P0kgxTEw/UserTweetsAndReplies', userId, {
+    paginationTweets(graphQLMap.UserTweetsAndReplies, userId, {
         ...params,
         withCommunity: true,
     });
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L825-L831
-const timelineMedia = (userId, params = {}) => paginationTweets('/graphql/nRybED9kRbN-TOWioHq1ng/UserMedia', userId, params);
+const timelineMedia = (userId, params = {}) => paginationTweets(graphQLMap.UserMedia, userId, params);
 
 // this query requires login
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L833-L839
-const timelineLikes = (userId, params = {}) => paginationTweets(`/graphql/9MSTt44HoGjVFSg_u3rHDw/Likes`, userId, params);
+const timelineLikes = (userId, params = {}) => paginationTweets(graphQLMap.Likes, userId, params);
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L858-L866
 const timelineKeywords = (keywords, params = {}) =>
-    twitterGot('https://twitter.com/i/api/2/search/adaptive.json', {
-        ..._params,
-        ...params,
-        q: keywords,
-        tweet_search_mode: 'live',
-        query_source: 'typed_query',
-        pc: 1,
+    got('https://api.twitter.com/1.1/search/universal.json', {
+        headers: {
+            Authorization: auth,
+        },
+        searchParams: {
+            ..._params,
+            ...params,
+            q: keywords,
+            modules: 'status',
+            result_type: 'recent',
+        },
     });
 
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L795-L805
 const tweetDetail = (userId, params) =>
     paginationTweets(
-        '/graphql/ItejhtHVxU7ksltgMmyaLA/TweetDetail',
+        graphQLMap.TweetDetail,
         userId,
         {
             ...params,
@@ -126,26 +134,21 @@ const tweetDetail = (userId, params) =>
         ['threaded_conversation_with_injections']
     );
 
-function gatherLegacyFromData(entries, filter = 'tweet-') {
+function gatherLegacyFromData(entries, filterNested = undefined, userId = undefined) {
     const tweets = [];
-    const filte_entries = [];
+    const filteredEntries = [];
     entries.forEach((entry) => {
         const entryId = entry.entryId;
         if (entryId) {
-            if (filter === 'none') {
-                if (entryId.startsWith('tweet-')) {
-                    filte_entries.push(entry);
-                } else if (entryId.startsWith('homeConversation-') || entryId.startsWith('conversationthread-')) {
-                    filte_entries.push(...entry.content.items);
-                }
-            } else {
-                if (entryId.startsWith(filter)) {
-                    filte_entries.push(entry);
-                }
+            if (entryId.startsWith('tweet-')) {
+                filteredEntries.push(entry);
+            }
+            if (filterNested && filterNested.some((f) => entryId.startsWith(f))) {
+                filteredEntries.push(...entry.content.items);
             }
         }
     });
-    filte_entries.forEach((entry) => {
+    filteredEntries.forEach((entry) => {
         if (entry.entryId) {
             const content = entry.content || entry.item;
             let tweet = content?.itemContent?.tweet_results?.result;
@@ -170,7 +173,9 @@ function gatherLegacyFromData(entries, filter = 'tweet-') {
                     if (retweet) {
                         legacy.retweeted_status = retweet.legacy;
                     }
-                    tweets.push(legacy);
+                    if (userId === undefined || legacy.user_id_str === userId + '') {
+                        tweets.push(legacy);
+                    }
                 }
             }
         }
@@ -178,40 +183,12 @@ function gatherLegacyFromData(entries, filter = 'tweet-') {
     return tweets;
 }
 
-function pickLegacyByID(id, tweets_dict, users_dict) {
-    function pickLegacyFromTweet(tweet) {
-        tweet.user = users_dict[tweet.user_id_str];
-        if (tweet.retweeted_status_id_str) {
-            tweet.retweeted_status = pickLegacyFromTweet(tweets_dict[tweet.retweeted_status_id_str]);
-        }
-        return tweet;
-    }
-
-    if (tweets_dict[id]) {
-        return pickLegacyFromTweet(tweets_dict[id]);
-    }
-}
-
-function gatherLegacyFromLegacyApiData(data, filter = 'tweet-') {
-    const tweets_dict = data.globalObjects.tweets;
-    const users_dict = data.globalObjects.users;
-    const tweets = [];
-    data.timeline.instructions[0].addEntries.entries.forEach((entry) => {
-        if (entry.entryId && entry.entryId.indexOf(filter) !== -1) {
-            const tweet = pickLegacyByID(entry.content.item.content.tweet.id, tweets_dict, users_dict);
-            if (tweet) {
-                tweets.push(tweet);
-            }
-        }
-    });
-    return tweets;
-}
-
 const getUserTweetsByID = async (id, params = {}) => gatherLegacyFromData(await timelineTweets(id, params));
-const getUserTweetsAndRepliesByID = async (id, params = {}) => gatherLegacyFromData(await timelineTweetsAndReplies(id, params));
+// TODO: show the whole conversation instead of just the reply tweet
+const getUserTweetsAndRepliesByID = async (id, params = {}) => gatherLegacyFromData(await timelineTweetsAndReplies(id, params), ['profile-conversation-'], id);
 const getUserMediaByID = async (id, params = {}) => gatherLegacyFromData(await timelineMedia(id, params));
 const getUserLikesByID = async (id, params = {}) => gatherLegacyFromData(await timelineLikes(id, params));
-const getUserTweetByStatus = async (id, params = {}) => gatherLegacyFromData(await tweetDetail(id, params), 'none');
+const getUserTweetByStatus = async (id, params = {}) => gatherLegacyFromData(await tweetDetail(id, params), ['homeConversation-', 'conversationthread-']);
 
 const excludeRetweet = function (tweets) {
     const excluded = [];
@@ -225,27 +202,80 @@ const excludeRetweet = function (tweets) {
 };
 
 const userByScreenName = (screenName) =>
-    twitterGot('https://twitter.com/i/api/graphql/hc-pka9A7gyS3xODIafnrQ/UserByScreenName', {
+    twitterGot(`https://twitter.com/i/api${graphQLMap.UserByScreenName}`, {
         variables: `{"screen_name":"${screenName}","withHighlightedLabel":true}`,
+        features: featuresMap.UserByScreenName,
+    });
+const userByRestId = (restId) =>
+    twitterGot(`https://twitter.com/i/api${graphQLMap.UserByRestId}`, {
+        variables: `{"userId":"${restId}","withHighlightedLabel":true}`,
+        features: featuresMap.UserByRestId,
     });
-const getUserData = (cache, screenName) => cache.tryGet(`twitter-userdata-${screenName}`, () => userByScreenName(screenName));
-const getUserID = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.rest_id;
-const getUser = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.legacy;
+const userByAuto = (id) => {
+    if (id.startsWith('+')) {
+        return userByRestId(id.slice(1));
+    }
+    return userByScreenName(id);
+};
+const getUserData = (cache, id) => cache.tryGet(`twitter-userdata-${id}`, () => userByAuto(id));
+const getUserID = async (cache, id) => (await getUserData(cache, id)).data.user.result.rest_id;
+const getUser = async (cache, id) => (await getUserData(cache, id)).data.user.result.legacy;
 
-const cacheTryGet = async (cache, screenName, params, func) => {
-    const id = await getUserID(cache, screenName);
+const cacheTryGet = async (cache, _id, params, func) => {
+    const id = await getUserID(cache, _id);
+    if (id === undefined) {
+        throw Error('User not found');
+    }
     const funcName = func.name;
     const paramsString = JSON.stringify(params);
     return cache.tryGet(`twitter:${id}:${funcName}:${paramsString}`, () => func(id, params), config.cache.routeExpire, false);
 };
 
-const getUserTweets = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserTweetsByID);
-const getUserTweetsAndReplies = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserTweetsAndRepliesByID);
-const getUserMedia = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserMediaByID);
-const getUserLikes = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserLikesByID);
-const getUserTweet = (cache, screenName, params) => cacheTryGet(cache, screenName, params, getUserTweetByStatus);
+// returns:
+// 1. nothing for some users
+// 2. HOT tweets for the other users, instead of the LATEST ones
+const _getUserTweets = (cache, id, params = {}) => cacheTryGet(cache, id, params, getUserTweetsByID);
+// workaround for the above issue:
+// 1. getUserTweetsAndReplies return LATEST tweets and replies, which requires filtering
+//    a. if one replies a lot (e.g. elonmusk), there is sometimes no tweets left after filtering, caching may help
+// 2. getUserMedia return LATEST media tweets, which is a good plus
+const getUserTweets = async (cache, id, params = {}) => {
+    let tweets = [];
+    const rest_id = await getUserID(cache, id);
+    await Promise.all(
+        [_getUserTweets, getUserTweetsAndReplies, getUserMedia].map(async (func) => {
+            try {
+                tweets.push(...(await func(cache, id, params)));
+            } catch (e) {
+                logger.warn(`Failed to get tweets for ${id} with ${func.name}: ${e}`);
+            }
+        })
+    );
+
+    const cacheKey = `twitter:user:tweets-cache:${rest_id}`;
+    let cacheValue = await cache.get(cacheKey);
+    if (cacheValue) {
+        cacheValue = JSON.parse(cacheValue);
+        if (cacheValue && cacheValue.length) {
+            tweets = cacheValue.concat(tweets);
+        }
+    }
+    const idSet = new Set();
+    tweets = tweets
+        .filter((tweet) => !tweet.in_reply_to_screen_name) // exclude replies
+        .map((tweet) => (idSet.has(tweet.id_str) ? null : idSet.add(tweet.id_str)) && tweet) // deduplicate
+        .filter(Boolean) // remove null
+        .sort((a, b) => b.id_str - a.id_str) // desc
+        .slice(0, 20);
+    cache.set(cacheKey, JSON.stringify(tweets));
+    return tweets;
+};
+const getUserTweetsAndReplies = (cache, id, params = {}) => cacheTryGet(cache, id, params, getUserTweetsAndRepliesByID);
+const getUserMedia = (cache, id, params = {}) => cacheTryGet(cache, id, params, getUserMediaByID);
+const getUserLikes = (cache, id, params = {}) => cacheTryGet(cache, id, params, getUserLikesByID);
+const getUserTweet = (cache, id, params) => cacheTryGet(cache, id, params, getUserTweetByStatus);
 
-const getSearch = async (keywords, params = {}) => gatherLegacyFromLegacyApiData(await timelineKeywords(keywords, params), 'sq-I-t-');
+const getSearch = async (keywords, params = {}) => (await timelineKeywords(keywords, params)).data.modules.map((module) => module.status.data);
 
 module.exports = {
     getUser,