rsshub 1.0.0-master.fe69819 → 1.0.0-master.febd623

package/lib/routes/twitter/web-api/twitter-api.js

@@ -1,4 +1,7 @@
 const twitterGot = require('./twitter-got');
+const config = require('@/config').value;
+
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L727-L755
 const _params = {
     count: 20,
     include_profile_interstitial_type: 1,
@@ -9,6 +12,7 @@ const _params = {
     include_mute_edge: 1,
     include_can_dm: 1,
     include_can_media_tag: 1,
+    include_ext_has_nft_avatar: 1,
     skip_status: 1,
     cards_platform: 'Web-12',
     include_cards: 1,
@@ -20,30 +24,83 @@ const _params = {
     include_user_entities: true,
     include_ext_media_color: true,
     include_ext_media_availability: true,
+    // include_ext_sensitive_media_warning: true,  // IDK what it is, maybe disabling it will make NSFW lovers happy?
     send_error_codes: true,
     simple_quoted_tweet: true,
     include_tweet_replies: false,
-    ext: 'mediaStats,highlightedLabel',
+    cursor: undefined,
+    ext: 'mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,superFollowMetadata',
 };
 
-const pagination = (endpoint, userId, params) =>
-    twitterGot('https://api.twitter.com' + endpoint, {
-        ..._params,
-        ...params,
-        userId,
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L756--L770
+const _variables = {
+    count: 20,
+    includePromotedContent: false,
+    withSuperFollowsUserFields: true,
+    withBirdwatchPivots: false,
+    withDownvotePerspective: false,
+    withReactionsMetadata: false,
+    withReactionsPerspective: false,
+    withSuperFollowsTweetFields: true,
+    withClientEventToken: false,
+    withBirdwatchNotes: false,
+    withVoice: true,
+    withV2Timeline: false,
+    __fs_interactive_text: false,
+    __fs_dont_mention_me_view_api_enabled: false,
+};
+
+// const paginationLegacy = (endpoint, userId, params) =>
+//     twitterGot('https://api.twitter.com' + endpoint, {
+//         ..._params,
+//         ...params,
+//         userId,
+//     });
+
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L1075-L1093
+const paginationTweets = async (endpoint, userId, variables, path) => {
+    const { data } = await twitterGot('https://twitter.com/i/api' + endpoint, {
+        variables: JSON.stringify({
+            ..._variables,
+            ...variables,
+            userId,
+        }),
     });
-const timelineMedia = (userId, params = {}) => pagination(`/2/timeline/media/${userId}.json`, userId, params);
-const timelineTweets = (userId, params = {}) => pagination(`/2/timeline/profile/${userId}.json`, userId, params);
-const timelineTweetsAndReplies = (userId, params = {}) =>
-    pagination(`/2/timeline/profile/${userId}.json`, userId, {
+
+    let instructions;
+    if (!path) {
+        instructions = data.user.result.timeline.timeline.instructions;
+    } else {
+        instructions = data;
+        path.forEach((p) => (instructions = instructions[p]));
+        instructions = instructions.instructions;
+    }
+
+    return instructions.filter((i) => i.type === 'TimelineAddEntries')[0].entries;
+};
+
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L807-L814
+const timelineTweets = (userId, params = {}) =>
+    paginationTweets('/graphql/WZT7sCTrLvSOaWOXLDsWbQ/UserTweets', userId, {
         ...params,
-        include_tweet_replies: true,
+        withQuickPromoteEligibilityTweetFields: true,
     });
-const timelineLikes = (userId, params = {}) =>
-    pagination(`/2/timeline/favorites/${userId}.json`, userId, {
+
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L816-L823
+const timelineTweetsAndReplies = (userId, params = {}) =>
+    paginationTweets('/graphql/t4wEKVulW4Mbv1P0kgxTEw/UserTweetsAndReplies', userId, {
         ...params,
-        sorted_by_time: true,
+        withCommunity: true,
     });
+
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L825-L831
+const timelineMedia = (userId, params = {}) => paginationTweets('/graphql/nRybED9kRbN-TOWioHq1ng/UserMedia', userId, params);
+
+// this query requires login
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L833-L839
+const timelineLikes = (userId, params = {}) => paginationTweets(`/graphql/9MSTt44HoGjVFSg_u3rHDw/Likes`, userId, params);
+
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L858-L866
 const timelineKeywords = (keywords, params = {}) =>
     twitterGot('https://twitter.com/i/api/2/search/adaptive.json', {
         ..._params,
@@ -53,6 +110,26 @@ const timelineKeywords = (keywords, params = {}) =>
         pc: 1,
     });
 
+function gatherLegacyFromData(entries, filter = 'tweet-') {
+    const tweets_dict = {};
+    const tweets = [];
+    entries.forEach((entry) => {
+        tweets_dict[entry.sortIndex] = entry;
+    });
+    entries.forEach((entry) => {
+        if (entry.entryId && entry.entryId.includes(filter)) {
+            const legacy = entry.content.itemContent.tweet_results.result.legacy;
+            legacy.user = entry.content.itemContent.tweet_results.result.core.user_results.result.legacy;
+            if (legacy.retweeted_status_result && legacy.retweeted_status_result.result) {
+                legacy.retweeted_status = legacy.retweeted_status_result.result.legacy;
+                legacy.retweeted_status.user = legacy.retweeted_status_result.result.core.user_results.result.legacy;
+            }
+            tweets.push(legacy);
+        }
+    });
+    return tweets;
+}
+
 function pickLegacyByID(id, tweets_dict, users_dict) {
     function pickLegacyFromTweet(tweet) {
         tweet.user = users_dict[tweet.user_id_str];
@@ -67,7 +144,7 @@ function pickLegacyByID(id, tweets_dict, users_dict) {
     }
 }
 
-function gatherLegacyFromData(data, filter = 'tweet-') {
+function gatherLegacyFromLegacyApiData(data, filter = 'tweet-') {
     const tweets_dict = data.globalObjects.tweets;
     const users_dict = data.globalObjects.users;
     const tweets = [];
@@ -106,11 +183,19 @@ const getUserData = (cache, screenName) => cache.tryGet(`twitter-userdata-${scre
 const getUserID = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.rest_id;
 const getUser = async (cache, screenName) => (await getUserData(cache, screenName)).data.user.legacy;
 
-const getUserTweets = async (cache, screenName, params = {}) => getUserTweetsByID(await getUserID(cache, screenName), params);
-const getUserTweetsAndReplies = async (cache, screenName, params = {}) => getUserTweetsAndRepliesByID(await getUserID(cache, screenName), params);
-const getUserMedia = async (cache, screenName, params = {}) => getUserMediaByID(await getUserID(cache, screenName), params);
-const getUserLikes = async (cache, screenName, params = {}) => getUserLikesByID(await getUserID(cache, screenName), params);
-const getSearch = async (keywords, params = {}) => gatherLegacyFromData(await timelineKeywords(keywords, params), 'sq-I-t-');
+const cacheTryGet = async (cache, screenName, params, func) => {
+    const id = await getUserID(cache, screenName);
+    const funcName = func.name;
+    const paramsString = JSON.stringify(params);
+    return await cache.tryGet(`twitter:${id}:${funcName}:${paramsString}`, async () => await func(id, params), config.cache.routeExpire, false);
+};
+
+const getUserTweets = async (cache, screenName, params = {}) => await cacheTryGet(cache, screenName, params, getUserTweetsByID);
+const getUserTweetsAndReplies = async (cache, screenName, params = {}) => await cacheTryGet(cache, screenName, params, getUserTweetsAndRepliesByID);
+const getUserMedia = async (cache, screenName, params = {}) => await cacheTryGet(cache, screenName, params, getUserMediaByID);
+const getUserLikes = async (cache, screenName, params = {}) => await cacheTryGet(cache, screenName, params, getUserLikesByID);
+
+const getSearch = async (keywords, params = {}) => gatherLegacyFromLegacyApiData(await timelineKeywords(keywords, params), 'sq-I-t-');
 
 module.exports = {
     getUser,

package/lib/routes/twitter/web-api/twitter-got.js

@@ -3,13 +3,17 @@ const { promisify } = require('util');
 const queryString = require('query-string');
 const got = require('@/utils/got');
 const config = require('@/config').value;
+// https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L716-L726
 const headers = {
     authorization: config.twitter.authorization,
+    // every single user and guest share the same token:
+    // Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA
     'x-guest-token': undefined,
     'x-twitter-auth-type': undefined,
     'x-twitter-client-language': 'en',
     'x-twitter-active-user': 'yes',
     'x-csrf-token': undefined,
+    Referer: 'https://twitter.com/',
 };
 let cookieJar, setCookie, getCookies;
 

package/lib/routes/twitter/web-api/user.js

@@ -5,14 +5,15 @@ module.exports = async (ctx) => {
     const id = ctx.params.id;
 
     // For compatibility
-    const { exclude_replies, include_rts } = utils.parseRouteParams(ctx.params.routeParams);
+    const { count, exclude_replies, include_rts } = utils.parseRouteParams(ctx.params.routeParams);
+    const params = count ? { count } : {};
 
     let data;
     const userInfo = await getUser(ctx.cache, id);
     if (exclude_replies) {
-        data = await getUserTweets(ctx.cache, id);
+        data = await getUserTweets(ctx.cache, id, params);
     } else {
-        data = await getUserTweetsAndReplies(ctx.cache, id);
+        data = await getUserTweetsAndReplies(ctx.cache, id, params);
     }
     if (!include_rts) {
         data = excludeRetweet(data);
@@ -23,7 +24,7 @@ module.exports = async (ctx) => {
     ctx.state.data = {
         title: `Twitter @${userInfo.name}`,
         link: `https://twitter.com/${id}`,
-        image: profileImageUrl,
+        image: profileImageUrl.replace(/_normal.jpg$/, '.jpg'),
         description: userInfo.description,
         item: utils.ProcessFeed(ctx, {
             data,

package/lib/routes/universities/hitwh/today.js

@@ -1,6 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const url = require('url');
+const { parseDate } = require('@/utils/parse-date');
 
 const baseUrl = 'http://today.hitwh.edu.cn';
 
@@ -9,10 +9,10 @@ module.exports = async (ctx) => {
     const { data } = response;
     const $ = cheerio.load(data);
     const type = (filename) => filename.split('.').pop();
-    const links = $('#righ_list ul li')
+    const links = $('.list_list_wrap > ul > li')
         .map((_, el) => ({
-            pubDate: new Date($(el).find('font').text().substr(-10, 10)).toUTCString(),
-            link: url.resolve(baseUrl, $('a', el).attr('href')),
+            pubDate: parseDate($(el).find('.news-time2').text()),
+            link: new URL($(el).find('a').attr('href'), baseUrl).toString(),
             title: $(el).find('a').text(),
         }))
         .get();
@@ -30,8 +30,8 @@ module.exports = async (ctx) => {
                             $('div.wp_articlecontent').html() &&
                             $('div.wp_articlecontent')
                                 .html()
-                                .replace(/src="\//g, `src="${url.resolve(baseUrl, '.')}`)
-                                .replace(/href="\//g, `href="${url.resolve(baseUrl, '.')}`)
+                                .replace(/src="\//g, `src="${baseUrl}/`)
+                                .replace(/href="\//g, `href="${baseUrl}/`)
                                 .trim();
                         return item;
                     } else {

package/lib/routes/weibo/keyword.js

@@ -3,21 +3,29 @@ const got = require('@/utils/got');
 const weiboUtils = require('./utils');
 const timezone = require('@/utils/timezone');
 const { fallback, queryToBoolean } = require('@/utils/readable-social');
+const config = require('@/config').value;
 
 module.exports = async (ctx) => {
     const keyword = ctx.params.keyword;
 
-    const response = await got({
-        method: 'get',
-        url: `https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D61%26q%3D${encodeURIComponent(keyword)}%26t%3D0`,
-        headers: {
-            Referer: `https://m.weibo.cn/p/searchall?containerid=100103type%3D1%26q%3D${encodeURIComponent(keyword)}`,
-            'MWeibo-Pwa': 1,
-            'X-Requested-With': 'XMLHttpRequest',
-            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
+    const data = await ctx.cache.tryGet(
+        `weibo:keyword:${keyword}`,
+        async () => {
+            const _r = await got({
+                method: 'get',
+                url: `https://m.weibo.cn/api/container/getIndex?containerid=100103type%3D61%26q%3D${encodeURIComponent(keyword)}%26t%3D0`,
+                headers: {
+                    Referer: `https://m.weibo.cn/p/searchall?containerid=100103type%3D1%26q%3D${encodeURIComponent(keyword)}`,
+                    'MWeibo-Pwa': 1,
+                    'X-Requested-With': 'XMLHttpRequest',
+                    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
+                },
+            });
+            return _r.data.data.cards;
         },
-    });
-    const data = response.data.data.cards;
+        config.cache.routeExpire,
+        false
+    );
 
     const routeParams = querystring.parse(ctx.params.routeParams);
     ctx.state.data = {

package/lib/routes/weibo/super_index.js

@@ -2,28 +2,37 @@ const got = require('@/utils/got');
 const weiboUtils = require('./utils');
 const timezone = require('@/utils/timezone');
 const queryString = require('query-string');
+const config = require('@/config').value;
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
     const type = ctx.params.type ?? 'feed';
 
-    const res = await got.get('https://m.weibo.cn/api/container/getIndex', {
-        searchParams: queryString.stringify({
-            containerid: `${id}_-_${type}`,
-            luicode: '10000011',
-            lfid: `${id}_-_main`,
-        }),
-        headers: {
-            Referer: `https://m.weibo.cn/p/index?containerid=${id}_-_soul&luicode=10000011&lfid=${id}_-_main`,
-            'MWeibo-Pwa': '1',
-            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
-            'X-Requested-With': 'XMLHttpRequest',
+    const containerData = await ctx.cache.tryGet(
+        `weibo:super_index:container:${id}:${type}`,
+        async () => {
+            const _r = await got('https://m.weibo.cn/api/container/getIndex', {
+                searchParams: queryString.stringify({
+                    containerid: `${id}_-_${type}`,
+                    luicode: '10000011',
+                    lfid: `${id}_-_main`,
+                }),
+                headers: {
+                    Referer: `https://m.weibo.cn/p/index?containerid=${id}_-_soul&luicode=10000011&lfid=${id}_-_main`,
+                    'MWeibo-Pwa': '1',
+                    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
+                    'X-Requested-With': 'XMLHttpRequest',
+                },
+            });
+            return _r.data.data;
         },
-    });
+        config.cache.routeExpire,
+        false
+    );
 
     const resultItems = [];
 
-    for (const card of res.data.data.cards) {
+    for (const card of containerData.cards) {
         if (!('card_group' in card)) {
             continue;
         }
@@ -45,9 +54,9 @@ module.exports = async (ctx) => {
     }
 
     ctx.state.data = {
-        title: `微博超话 - ${res.data.data.pageInfo.page_title}`,
+        title: `微博超话 - ${containerData.pageInfo.page_title}`,
         link: `https://weibo.com/p/${id}/super_index`,
-        description: `#${res.data.data.pageInfo.page_title}# 的超话`,
+        description: `#${containerData.pageInfo.page_title}# 的超话`,
         item: resultItems,
     };
 };

package/lib/routes/weibo/timeline.js

@@ -11,30 +11,50 @@ module.exports = async (ctx) => {
     const routeParams = ctx.params.routeParams || undefined;
     const token = await ctx.cache.get('weibotimelineuid' + uid, false);
     let displayVideo = '1';
+    let displayArticle = '0';
+    let displayComments = '0';
     if (routeParams) {
         if (routeParams === '1' || routeParams === '0') {
             displayVideo = routeParams;
         } else {
             const routeParams = querystring.parse(ctx.params.routeParams);
             displayVideo = fallback(undefined, queryToBoolean(routeParams.displayVideo), true) ? '1' : '0';
+            displayArticle = fallback(undefined, queryToBoolean(routeParams.displayArticle), false) ? '1' : '0';
+            displayComments = fallback(undefined, queryToBoolean(routeParams.displayComments), false) ? '1' : '0';
         }
     }
 
     if (token) {
-        const containerResponse = await got({
-            method: 'get',
-            url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
-            headers: {
-                Referer: 'https://m.weibo.cn/',
+        const userInfo = await ctx.cache.tryGet(
+            `weibo:timeline:userInfo:${uid}`,
+            async () => {
+                const _r = await got({
+                    method: 'get',
+                    url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
+                    headers: {
+                        Referer: 'https://m.weibo.cn/',
+                    },
+                });
+                return _r.data.data.userInfo;
             },
-        });
-        const name = containerResponse.data.data.userInfo.screen_name;
-        const description = containerResponse.data.data.userInfo.description;
-        const profileImageUrl = containerResponse.data.data.userInfo.profile_image_url;
+            config.cache.routeExpire,
+            false
+        );
+        const name = userInfo.screen_name;
+        const description = userInfo.description;
+        const profileImageUrl = userInfo.profile_image_url;
 
-        const response = await got.get(`https://api.weibo.com/2/statuses/home_timeline.json?access_token=${token}&count=100&feature=${feature}`);
+        const response = await ctx.cache.tryGet(
+            `weibo:timeline:${uid}`,
+            async () => {
+                const _r = await got(`https://api.weibo.com/2/statuses/home_timeline.json?access_token=${token}&count=100&feature=${feature}`);
+                return _r.data;
+            },
+            config.cache.routeExpire,
+            false
+        );
         // 检查token失效
-        if (response.data.error !== undefined) {
+        if (response.error !== undefined) {
             const { app_key = '', redirect_url = ctx.request.origin + '/weibo/timeline/0' } = config.weibo;
 
             ctx.status = 302;
@@ -44,13 +64,9 @@ module.exports = async (ctx) => {
             ctx.redirect(`https://api.weibo.com/oauth2/authorize?client_id=${app_key}&redirect_uri=${redirect_url}${routeParams ? `&state=${routeParams}` : ''}`);
         }
         const resultItem = await Promise.all(
-            response.data.statuses.map(async (item) => {
+            response.statuses.map(async (item) => {
                 const key = `weibotimelineurl${item.user.id}${item.id}`;
-                const value = await ctx.cache.tryGet(key, async () => {
-                    const _data = await weiboUtils.getShowData(uid, item.id);
-                    return JSON.stringify(_data);
-                });
-                const data = JSON.parse(value);
+                const data = await ctx.cache.tryGet(key, () => weiboUtils.getShowData(uid, item.id));
 
                 // 是否通过api拿到了data
                 const isDataOK = data?.text;
@@ -61,7 +77,7 @@ module.exports = async (ctx) => {
                 // 转发的长微博处理
                 const retweet = item.retweeted_status;
                 if (retweet?.isLongText) {
-                    const retweetData = await weiboUtils.getShowData(retweet.user.id, retweet.id);
+                    const retweetData = await ctx.cache.tryGet(`weibo:retweeted:${retweet.user.id}:${retweet.id}`, () => weiboUtils.getShowData(retweet.user.id, retweet.id));
                     if (retweetData?.text) {
                         item.retweeted_status.text = retweetData.text;
                     }
@@ -76,7 +92,27 @@ module.exports = async (ctx) => {
 
                 // 视频的处理
                 if (displayVideo === '1') {
-                    description = weiboUtils.formatVideo(description, item);
+                    // 含被转发微博时需要从被转发微博中获取视频
+                    if (item.retweeted_status) {
+                        description = weiboUtils.formatVideo(description, item.retweeted_status);
+                    } else {
+                        description = weiboUtils.formatVideo(description, item);
+                    }
+                }
+
+                // 评论的处理
+                if (displayComments === '1') {
+                    description = await weiboUtils.formatComments(ctx, description, item);
+                }
+
+                // 文章的处理
+                if (displayArticle === '1') {
+                    // 含被转发微博时需要从被转发微博中获取文章
+                    if (item.retweeted_status) {
+                        description = await weiboUtils.formatArticle(ctx, description, item.retweeted_status);
+                    } else {
+                        description = await weiboUtils.formatArticle(ctx, description, item);
+                    }
                 }
 
                 return {
@@ -106,7 +142,7 @@ module.exports = async (ctx) => {
             const token = rep.data.access_token;
             const uid = rep.data.uid;
             const expires_in = rep.data.expires_in;
-            await ctx.cache.set('weibotimelineuid' + uid, token, expires_in, false);
+            await ctx.cache.set('weibotimelineuid' + uid, token, expires_in);
 
             ctx.set({
                 'Content-Type': 'text/html; charset=UTF-8',