rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/routes/weibo/user.js

@@ -1,6 +1,7 @@
-const querystring = require('querystring');
+const querystring = require('query-string');
 const got = require('@/utils/got');
 const weiboUtils = require('./utils');
+const config = require('@/config').value;
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const { fallback, queryToBoolean } = require('@/utils/readable-social');
@@ -9,6 +10,7 @@ module.exports = async (ctx) => {
     const uid = ctx.params.uid;
     let displayVideo = '1';
     let displayArticle = '0';
+    let displayComments = '0';
     if (ctx.params.routeParams) {
         if (ctx.params.routeParams === '1' || ctx.params.routeParams === '0') {
             displayVideo = ctx.params.routeParams;
@@ -16,47 +18,58 @@ module.exports = async (ctx) => {
             const routeParams = querystring.parse(ctx.params.routeParams);
             displayVideo = fallback(undefined, queryToBoolean(routeParams.displayVideo), true) ? '1' : '0';
             displayArticle = fallback(undefined, queryToBoolean(routeParams.displayArticle), false) ? '1' : '0';
+            displayComments = fallback(undefined, queryToBoolean(routeParams.displayComments), false) ? '1' : '0';
         }
     }
 
-    const { data: containerData } = await got({
-        method: 'get',
-        url: `https://m.weibo.cn/profile/info?uid=${uid}`,
-        headers: {
-            Referer: 'https://m.weibo.cn/',
+    const containerData = await ctx.cache.tryGet(
+        `weibo:user:index:${uid}`,
+        async () => {
+            const _r = await got({
+                method: 'get',
+                url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
+                headers: {
+                    Referer: `https://m.weibo.cn/u/${uid}`,
+                    'MWeibo-Pwa': 1,
+                    'X-Requested-With': 'XMLHttpRequest',
+                },
+            });
+            return _r.data;
         },
-    });
-    const name = containerData.data.user.screen_name;
-    const description = containerData.data.user.description;
-    const profileImageUrl = containerData.data.user.profile_image_url;
-    const containerid = containerData.data.more.match(/\d+/)[0];
+        config.cache.routeExpire,
+        false
+    );
+    const name = containerData.data.userInfo.screen_name;
+    const description = containerData.data.userInfo.description;
+    const profileImageUrl = containerData.data.userInfo.profile_image_url;
+    const containerId = containerData.data.tabsInfo.tabs.filter((item) => item.tab_type === 'weibo')[0].containerid;
 
-    const response = await got({
-        method: 'get',
-        url: `https://m.weibo.cn/api/container/getIndex?containerid=${containerid}`,
-        headers: {
-            Referer: `https://m.weibo.cn/u/${uid}`,
-            'MWeibo-Pwa': 1,
-            'X-Requested-With': 'XMLHttpRequest',
+    const cards = await ctx.cache.tryGet(
+        `weibo:user:cards:${uid}:${containerId}`,
+        async () => {
+            const _r = await got({
+                method: 'get',
+                url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`,
+                headers: {
+                    Referer: `https://m.weibo.cn/u/${uid}`,
+                    'MWeibo-Pwa': 1,
+                    'X-Requested-With': 'XMLHttpRequest',
+                },
+            });
+            return _r.data.data.cards;
         },
-    });
-
-    let earliestDate;
+        config.cache.routeExpire,
+        false
+    );
 
-    const resultItem = await Promise.all(
-        response.data.data.cards
-            .reverse() // reverse first to ensure the pinned card will be processed lastly
+    let resultItems = await Promise.all(
+        cards
+            .filter((item) => item.mblog)
             .map(async (item) => {
-                if (!item.mblog && item.card_group && Array.isArray(item.card_group) && item.card_group.length > 0) {
-                    // w/o `mblog`, it may have a `card_group` containing only 1 card
-                    // only get the 1st card because we haven't observed that there can be multiple cards in it
-                    item = item.card_group[0];
-                }
-
-                if (!item.mblog) {
-                    return; // drop
-                }
-
+                // TODO: unify cache key and let weiboUtils.getShowData() handle the cache? It seems safe to do so.
+                //       Need more investigation, pending for now since the current version works fine.
+                // TODO: getShowData() on demand? The API seems to return most things we need since 2022/05/21.
+                //       Need more investigation, pending for now since the current version works fine.
                 const key = 'weibo:user:' + item.mblog.bid;
                 const data = await ctx.cache.tryGet(key, () => weiboUtils.getShowData(uid, item.mblog.bid));
 
@@ -71,21 +84,11 @@ module.exports = async (ctx) => {
                     item.mblog.created_at = timezone(item.mblog.created_at, +8);
                 }
 
-                // drop too old pinned weibo, otherwise preserve it
-                if (!item.mblog.title || item.mblog.title.text !== '置顶') {
-                    if (!earliestDate || (item.mblog.created_at && earliestDate > item.mblog.created_at)) {
-                        earliestDate = item.mblog.created_at;
-                    }
-                } else {
-                    if (earliestDate && item.mblog.created_at && earliestDate > item.mblog.created_at) {
-                        return;
-                    }
-                }
-
                 // 转发的长微博处理
                 const retweet = item.mblog.retweeted_status;
                 if (retweet && retweet.isLongText) {
-                    const retweetData = await weiboUtils.getShowData(retweet.user.id, retweet.bid);
+                    // TODO: unify cache key and ...
+                    const retweetData = await ctx.cache.tryGet(`weibo:retweeted:${retweet.user.id}:${retweet.bid}`, () => weiboUtils.getShowData(retweet.user.id, retweet.bid));
                     if (retweetData !== undefined && retweetData.text) {
                         item.mblog.retweeted_status.text = retweetData.text;
                     }
@@ -107,6 +110,11 @@ module.exports = async (ctx) => {
                     }
                 }
 
+                // 评论的处理
+                if (displayComments === '1') {
+                    description = await weiboUtils.formatComments(ctx, description, item.mblog);
+                }
+
                 // 文章的处理
                 if (displayArticle === '1') {
                     // 含被转发微博时需要从被转发微博中获取文章
@@ -123,15 +131,25 @@ module.exports = async (ctx) => {
                     link,
                     pubDate,
                     author: item.mblog.user.screen_name,
+                    isPinned: item.profile_type_id?.startsWith('proweibotop'),
                 };
             })
     );
 
+    // remove pinned weibo if they are too old (older than all the rest weibo)
+    // the character of pinned weibo is `card.profile_type_id.startsWith('proweibotop')`
+    // there can be 1 or 2 (WHAT A FANTASTIC BRAIN THE PM HAS?) pinned weibo at the same time
+    const pinnedItems = resultItems.filter((item) => item.isPinned);
+    const ordinaryItems = resultItems.filter((item) => !item.isPinned);
+    if (pinnedItems.length > 0 && ordinaryItems.length > 0 && Math.max(...pinnedItems.map((i) => i.pubDate).filter(Boolean)) < Math.min(...ordinaryItems.map((i) => i.pubDate).filter(Boolean))) {
+        resultItems = ordinaryItems;
+    }
+
     ctx.state.data = {
         title: `${name}的微博`,
-        link: `http://weibo.com/${uid}/`,
+        link: `https://weibo.com/${uid}/`,
         description,
         image: profileImageUrl,
-        item: resultItem.filter((item) => item).reverse(),
+        item: resultItems,
     };
 };

package/lib/routes/weibo/utils.js

@@ -24,6 +24,7 @@ const weiboUtils = {
             heightOfPics: fallback(params.heightOfPics, queryToInteger(routeParams.heightOfPics), -1),
             sizeOfAuthorAvatar: fallback(params.sizeOfAuthorAvatar, queryToInteger(routeParams.sizeOfAuthorAvatar), 48),
             showEmojiInDescription: fallback(params.showEmojiInDescription, queryToInteger(routeParams.showEmojiInDescription), true),
+            showLinkIconInDescription: fallback(params.showLinkIconInDescription, queryToInteger(routeParams.showLinkIconInDescription), true),
         };
 
         params = mergedParams;
@@ -44,6 +45,7 @@ const weiboUtils = {
             heightOfPics,
             sizeOfAuthorAvatar,
             showEmojiInDescription,
+            showLinkIconInDescription,
         } = params;
 
         let retweeted = '';
@@ -51,12 +53,16 @@ const weiboUtils = {
         let htmlNewLineUnreplaced = (status.longText && status.longText.longTextContent) || status.text || '';
         // 表情图标转换为文字
         if (!showEmojiInDescription) {
-            htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/<span class=["']url-icon["']><img\s[^>]*?alt=["']?([^>]+?)["']?\s[^>]*?\/><\/span>/g, '$1');
+            htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/<span class=["']?url-icon["']?><img\s[^>]*?alt=["']?([^>]+?)["']?\s[^>]*?\/><\/span>/g, '$1');
         }
-        // 去掉外部链接的图标，保留 a 标签链接  // 不能去掉，否则像超话这样的有意义的图标也会被去掉
-        // htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/(<a\s[^>]*>)<span class=["']url-icon["']><img\s[^>]*><\/span>[^<>]*?<span class="surl-text">([^<>]*?)<\/span><\/a>/g, '$1$2</a>');
-        // 去掉乱七八糟的图标  // 不能去掉，否则像超话这样的有意义的图标也会被去掉
-        // htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/<span class=["']url-icon["']>(<img\s[^>]*?>)<\/span>/g, '');
+        // 去掉链接的图标，保留 a 标签链接
+        if (!showLinkIconInDescription) {
+            htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/(<a\s[^>]*>)<span class=["']?url-icon["']?><img\s[^>]*><\/span>[^<>]*?<span class=["']?surl-text["']?>([^<>]*?)<\/span><\/a>/g, '$1$2</a>');
+        }
+        // 去掉乱七八糟的图标  // 不需要，上述的替换应该已经把所有的图标都替换掉了，且这条 regex 会破坏上述替换不发生时的输出
+        // htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/<span class=["']?url-icon["']?>(<img\s[^>]*?>)<\/span>/g, '');
+        // 将行内图标的高度设置为一行，改善阅读体验。但有些阅读器删除了 style 属性，无法生效  // 不需要，微博已经作此设置
+        // htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/(?<=<span class=["']?url-icon["']?>)<img/g, '<img style="height: 1em"');
         // 去掉全文
         htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/全文<br>/g, '<br>');
         htmlNewLineUnreplaced = htmlNewLineUnreplaced.replace(/<a href="(.*?)">全文<\/a>/g, '');
@@ -100,8 +106,12 @@ const weiboUtils = {
             }
         }
 
+        // drop live photo
+        const livePhotoCount = status.pics ? status.pics.filter((pic) => pic.type === 'livephotos').length : 0;
+        const pics = status.pics && status.pics.filter((pic) => pic.type !== 'livephotos');
+
         // 添加微博配图
-        if (status.pics) {
+        if (pics) {
             if (readable) {
                 html += '<br clear="both" /><div style="clear: both"></div>';
             }
@@ -109,12 +119,12 @@ const weiboUtils = {
             // 一些RSS Reader会识别所有<img>标签作为内含图片显示，我们不想要头像也作为内含图片之一
             // 让所有配图在description的最前面再次出现一次，但宽高设为0
             let picsPrefix = '';
-            status.pics.forEach((item) => {
+            pics.forEach((item) => {
                 picsPrefix += `<img width="0" height="0" hidden="true" src="${item.large.url}">`;
             });
             picsPrefixes.push(picsPrefix);
 
-            status.pics.forEach((item) => {
+            pics.forEach((item) => {
                 if (addLinkForPics) {
                     html += '<a href="' + item.large.url + '">';
                 }
@@ -195,7 +205,8 @@ const weiboUtils = {
                 .replace(/<span class=["']url-icon["']>(<img\s[^>]*?>)<\/span>/g, '') // 去掉所有图标
                 .replace(/<img[\s\S]*?>/g, '[图片]')
                 .replace(/<.*?>/g, '')
-                .replace('\n', '');
+                .replace('\n', '')
+                .trim();
         }
         if (status.retweeted_status) {
             title += showEmojiForRetweet ? '🔁 ' : ' - 转发 ';
@@ -204,7 +215,15 @@ const weiboUtils = {
                 .replace(/<span class=["']url-icon["']>(<img\s[^>]*?>)<\/span>/g, '') // 去掉所有图标
                 .replace(/<img[\s\S]*?>/g, '[图片]')
                 .replace(/<.*?>/g, '')
-                .replace('\n', '');
+                .replace('\n', '')
+                .trim();
+        }
+        if (livePhotoCount > 0) {
+            title += ' ';
+            title += new Array(livePhotoCount + 1).join('[Live Photo]');
+        }
+        if (status.page_info && status.page_info === 'video') {
+            title += ' [视频]';
         }
 
         return { description: html, title };
@@ -223,6 +242,15 @@ const weiboUtils = {
     },
     formatVideo: (itemDesc, status) => {
         const pageInfo = status.page_info;
+        const livePhotos = status.pics && status.pics.filter((pic) => pic.type === 'livephotos' && pic.videoSrc);
+        let video = '<br clear="both" /><div style="clear: both"></div>';
+        let anyVideo = false;
+        if (livePhotos) {
+            livePhotos.forEach((livePhoto) => {
+                video += `<video controls="controls" poster="${(livePhoto.large && livePhoto.large.url) || livePhoto.url}" src="${livePhoto.videoSrc}" style="width: 100%"></video>`;
+                anyVideo = true;
+            });
+        }
         if (pageInfo && pageInfo.type === 'video') {
             const pagePic = pageInfo.page_pic;
             const posterUrl = pagePic ? pagePic.url : '';
@@ -238,7 +266,6 @@ const weiboUtils = {
             const hasVideo = video720p || videoHd || videoHdHevc || videoLd;
 
             if (hasVideo) {
-                let video = '<br clear="both" /><div style="clear: both"></div>';
                 video += `<video controls="controls" poster="${posterUrl}" style="width: 100%">`;
                 if (video720p) {
                     video += `<source src="${video720p}">`;
@@ -256,9 +283,12 @@ const weiboUtils = {
                     video += `<p>视频无法显示，请前往<a href="${pageUrl}" target="_blank" rel="noopener noreferrer">微博视频</a>观看。</p>`;
                 }
                 video += '</video>';
-                itemDesc += video;
+                anyVideo = true;
             }
         }
+        if (anyVideo) {
+            itemDesc += video;
+        }
         return itemDesc;
     },
     formatArticle: async (ctx, itemDesc, status) => {
@@ -348,6 +378,45 @@ const weiboUtils = {
         }
         return itemDesc;
     },
+    formatComments: async (ctx, itemDesc, status) => {
+        if (status && status.comments_count && status.id && status.mid) {
+            const id = status.id;
+            const mid = status.mid;
+            const link = `https://m.weibo.cn/comments/hotflow?id=${id}&mid=${mid}&max_id_type=0`;
+            const response = await ctx.cache.tryGet(link, async () => {
+                const _response = await got.get(link, {
+                    headers: {
+                        Referer: `https://m.weibo.cn/detail/${id}`,
+                        'MWeibo-Pwa': 1,
+                        'X-Requested-With': 'XMLHttpRequest',
+                        'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
+                    },
+                });
+                return _response.data;
+            });
+            if (response.data && response.data.data) {
+                const comments = response.data.data;
+                itemDesc += `<br clear="both" /><div style="clear: both"></div><div style="background: #80808010;border-top:1px solid #80808030;border-bottom:1px solid #80808030;margin:0;padding:5px 20px;">`;
+                itemDesc += '<h3>热门评论</h3>';
+                comments.forEach((comment) => {
+                    itemDesc += '<p style="margin-bottom: 0.5em;margin-top: 0.5em">';
+                    itemDesc += `<a href="https://weibo.com/${comment.user.id}" target="_blank">${comment.user.screen_name}</a>: ${comment.text}`;
+                    if (comment.comments) {
+                        itemDesc += '<blockquote style="border-left:0.2em solid #80808080; margin-left: 0.3em; padding-left: 0.5em; margin-bottom: 0.5em; margin-top: 0.25em">';
+                        comment.comments.forEach((comment) => {
+                            itemDesc += '<div style="font-size: 0.9em">';
+                            itemDesc += `<a href="https://weibo.com/${comment.user.id}" target="_blank">${comment.user.screen_name}</a>: ${comment.text}`;
+                            itemDesc += '</div>';
+                        });
+                        itemDesc += '</blockquote>';
+                    }
+                    itemDesc += '</p>';
+                });
+                itemDesc += '</div>';
+            }
+        }
+        return itemDesc;
+    },
 };
 
 module.exports = weiboUtils;

package/lib/routes/ximalaya/album.js

@@ -179,7 +179,7 @@ module.exports = async (ctx) => {
         const itunesItemImage = item.coverLarge ? item.coverLarge : albumCover;
         const link = baseUrl + albumUrl + trackId;
         const pubDate = new Date(item.createdAt).toUTCString();
-        const enclosureLength = item.duration; // 时间长度：单位（秒）
+        const duration = item.duration; // 时间长度：单位（秒）
         const enclosureUrl = item.playPathAacv224 || item.playPathAacv164;
 
         let resultItem = {
@@ -199,7 +199,7 @@ module.exports = async (ctx) => {
             resultItem = {
                 ...resultItem,
                 enclosure_url: enclosureUrl,
-                enclosure_length: enclosureLength,
+                itunes_duration: duration,
                 enclosure_type: 'audio/x-m4a',
             };
         }

package/lib/routes/zcool/top.js

@@ -8,7 +8,7 @@ module.exports = async (ctx) => {
     const topResponse = await got({
         method: 'get',
         url,
-        header: {
+        headers: {
             Referer: 'https://www.zcool.com.cn',
         },
     });
@@ -20,7 +20,7 @@ module.exports = async (ctx) => {
         const articleResponse = await got({
             method: 'get',
             url: 'https://www.zcool.com.cn/top/article.do?rankType=8&rankId=' + rankId,
-            header: {
+            headers: {
                 Referer: url,
             },
         });
@@ -56,7 +56,7 @@ module.exports = async (ctx) => {
             const itemResponse = await got({
                 method: 'get',
                 url: link,
-                header: {
+                headers: {
                     Referer: url,
                 },
             });

package/lib/utils/cf-email.js

@@ -0,0 +1,36 @@
+/**
+ * The alogrithm is well-explained in https://andrewlock.net/simple-obfuscation-of-email-addresses-using-javascript/
+ */
+module.exports = {
+    // eslint-disable-next-line lines-around-comment
+    /**
+     * Returns decoded email address using CloudFlare's email address obfuscation.
+     * @param {String} encoded - encoded email, (`cfemail` attribute in `__cf_email__` tag)
+     * @returns decoded email address
+     */
+    decodeCFEmail: (encoded) => {
+        const parseHex = (string, position) => parseInt(string.substr(position, 2), 16);
+        let decoded = '';
+        const key = parseHex(encoded, 0);
+        for (let position = 2; position < encoded.length; position += 2) {
+            const byte = parseHex(encoded, position) ^ key;
+            decoded += String.fromCharCode(byte);
+        }
+        return decoded;
+    },
+
+    /**
+     * Returns CloudFlare protected email address.
+     * @param {String} email - email in plaintext
+     * @returns obfuscated email
+     */
+    encodeCFEmail: (email) => {
+        const key = Math.floor(Math.random() * 256);
+        let encoded = ('0' + key.toString(16)).slice(-2);
+        for (let position = 0; position < email.length; position++) {
+            const byte = email.charCodeAt(position) ^ key;
+            encoded += ('0' + byte.toString(16)).slice(-2);
+        }
+        return encoded;
+    },
+};

package/lib/utils/common-config.js

@@ -3,6 +3,10 @@ const got = require('@/utils/got');
 const iconv = require('iconv-lite');
 // eslint-disable-next-line no-unused-vars
 const date = require('@/utils/date');
+// eslint-disable-next-line no-unused-vars
+const { parseDate } = require('@/utils/parse-date');
+// eslint-disable-next-line no-unused-vars
+const timezone = require('@/utils/timezone');
 
 function transElemText($, prop) {
     const regex = new RegExp(/\$\((.*)\)/g);

package/lib/utils/common-utils.js

@@ -1,3 +1,5 @@
+const { parseDate } = require('@/utils/parse-date');
+
 // convert a string into title case
 const toTitleCase = (str) =>
     str
@@ -6,6 +8,30 @@ const toTitleCase = (str) =>
         .map((word) => word.replace(word[0], word[0].toUpperCase()))
         .join(' ');
 
+const rWhiteSpace = /\s+/;
+const rAllWhiteSpace = /\s+/g;
+
+// collapse all whitespaces into a single space (like "white-space: normal;" would do), and trim
+const collapseWhitespace = (str) => {
+    if (str && rWhiteSpace.test(str)) {
+        return str.replace(rAllWhiteSpace, ' ').trim();
+    }
+    return str;
+};
+
+const convertDateToISO8601 = (date) => {
+    if (!date) {
+        return date;
+    }
+    if (typeof date !== 'object') {
+        // some routes may call `.toUTCString()` before passing the date to ctx...
+        date = parseDate(date);
+    }
+    return date.toISOString();
+};
+
 module.exports = {
     toTitleCase,
+    collapseWhitespace,
+    convertDateToISO8601,
 };