rsshub 1.0.0-master.ff1cc11 → 1.0.0-master.ff97759

package/lib/routes/weibo/timeline.js

@@ -1,170 +1,154 @@
+const querystring = require('querystring');
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 const config = require('@/config').value;
+const weiboUtils = require('./utils');
+const { fallback, queryToBoolean } = require('@/utils/readable-social');
 
 module.exports = async (ctx) => {
     const uid = ctx.params.uid;
     const feature = ctx.params.feature || 0;
+    const routeParams = ctx.params.routeParams || undefined;
     const token = await ctx.cache.get('weibotimelineuid' + uid, false);
+    let displayVideo = '1';
+    let displayArticle = '0';
+    let displayComments = '0';
+    if (routeParams) {
+        if (routeParams === '1' || routeParams === '0') {
+            displayVideo = routeParams;
+        } else {
+            const routeParams = querystring.parse(ctx.params.routeParams);
+            displayVideo = fallback(undefined, queryToBoolean(routeParams.displayVideo), true) ? '1' : '0';
+            displayArticle = fallback(undefined, queryToBoolean(routeParams.displayArticle), false) ? '1' : '0';
+            displayComments = fallback(undefined, queryToBoolean(routeParams.displayComments), false) ? '1' : '0';
+        }
+    }
 
     if (token) {
-        const response = await got.get(`https://api.weibo.com/2/statuses/home_timeline.json?access_token=${token}&count=100&feature=${feature}`);
+        const userInfo = await ctx.cache.tryGet(
+            `weibo:timeline:userInfo:${uid}`,
+            async () => {
+                const _r = await got({
+                    method: 'get',
+                    url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
+                    headers: {
+                        Referer: 'https://m.weibo.cn/',
+                    },
+                });
+                return _r.data.data.userInfo;
+            },
+            config.cache.routeExpire,
+            false
+        );
+        const name = userInfo.screen_name;
+        const description = userInfo.description;
+        const profileImageUrl = userInfo.profile_image_url;
+
+        const response = await ctx.cache.tryGet(
+            `weibo:timeline:${uid}`,
+            async () => {
+                const _r = await got(`https://api.weibo.com/2/statuses/home_timeline.json?access_token=${token}&count=100&feature=${feature}`);
+                return _r.data;
+            },
+            config.cache.routeExpire,
+            false
+        );
         // 检查token失效
-        if (response.data.error !== undefined) {
+        if (response.error !== undefined) {
             const { app_key = '', redirect_url = ctx.request.origin + '/weibo/timeline/0' } = config.weibo;
 
             ctx.status = 302;
             ctx.set({
                 'Cache-Control': 'no-cache',
             });
-            ctx.redirect(`https://api.weibo.com/oauth2/authorize?client_id=${app_key}&redirect_uri=${redirect_url}`);
+            ctx.redirect(`https://api.weibo.com/oauth2/authorize?client_id=${app_key}&redirect_uri=${redirect_url}${routeParams ? `&state=${routeParams}` : ''}`);
         }
-        // 获取所有表情
-        const weiboEmotion = await ctx.cache.tryGet('weiboEmotion', async () => {
-            const tmpEmotionList = {};
-            await Promise.all(
-                ['face', 'ani', 'cartoon'].map(async (type) => {
-                    try {
-                        const rep = await got.get(`https://api.weibo.com/2/emotions.json?access_token=${token}&type=${type}`);
-                        rep.data.forEach((item) => {
-                            tmpEmotionList[item.phrase] = item.url;
-                        });
-                        return Promise.resolve(rep);
-                    } catch (err) {
-                        // pass
-                    }
-                })
-            );
-            return tmpEmotionList;
-        });
+        const resultItem = await Promise.all(
+            response.statuses.map(async (item) => {
+                const key = `weibotimelineurl${item.user.id}${item.id}`;
+                const data = await ctx.cache.tryGet(key, () => weiboUtils.getShowData(uid, item.id));
 
-        const data = await Promise.all(
-            response.data.statuses.map(async (item) => {
-                const realContent = await ctx.cache.tryGet('weibotimelineurl' + item.user.id + item.id, async () => {
-                    const res = [];
-                    const videoList = [];
-                    let picUrlList = [];
-                    picUrlList = picUrlList.concat(item.pic_urls);
-                    try {
-                        const rep = await got.get(`https://m.weibo.cn/${item.user.id}/${item.id}`);
-                        rep.body.replace(/"text": "([\s\S]*?)",/g, (s, value) => {
-                            res.push(value.replace(/\\"/g, "'"));
-                        });
-                        rep.body.replace(/"mp4_720p_mp4": "([\s\S]*?)",/, (s, value) => {
-                            videoList.push(value);
-                        });
-                        if (res.length > 1) {
-                            const sub_rep = await got.get(`https://m.weibo.cn/${item.retweeted_status.user.id}/${item.retweeted_status.id}`);
-                            sub_rep.body.replace(/"text": "([\s\S]*?)",/g, (s, value) => {
-                                res[1] = value.replace(/\\"/g, "'");
-                            });
-                            sub_rep.body.replace(/"mp4_720p_mp4": "([\s\S]*?)",/, (s, value) => {
-                                videoList.push(value);
-                            });
-                            picUrlList = picUrlList.concat(item.retweeted_status.pic_urls);
-                        }
-                    } catch (err) {
-                        res.push(item.text);
-                        if (item.retweeted_status) {
-                            res.push(item.retweeted_status.text);
-                            picUrlList = picUrlList.concat(item.retweeted_status.pic_urls);
-                        }
-                        // 非全文获取的微博内容需要替换表情
-                        res.forEach((content, index) => {
-                            const emotion = [];
-                            content.replace(/(\[.*?\])/g, (s, value) => {
-                                emotion.push(value);
-                            });
-
-                            if (content.length > 0) {
-                                emotion.forEach((item) => {
-                                    res[index] = content.replace(new RegExp(`\\${item}`, 'g'), `<img alt="${item}" src="${weiboEmotion[item]}" style="width:1em; height:1em;" referrerpolicy="no-referrer">`);
-                                });
-                            }
-                        });
-                    }
-                    let content = '';
-                    try {
-                        if (res.length >= 2) {
-                            content = `${res[0]}<br><br><hr style="height:5px;border:none;border-top:5px ridge green;" /><span style="background-color:#71adec">${item.retweeted_status.user.screen_name}:<hr style="height:1px;border:none;border-top:1px solid #555555;" /><br>${res[1]}</span>`;
-                        } else if (res.length === 1) {
-                            content = res[0];
-                        }
-
-                        // 手机端reeder查看内容时，表情会导致换行-此处做转换
-                        content.replace(/<span class='url-icon'>([\s\S]*?)<\/span>/g, (s, value) => {
-                            content = content.replace(s, value);
-                        });
-                    } catch (error) {
-                        // pass
+                // 是否通过api拿到了data
+                const isDataOK = data?.text;
+                if (isDataOK) {
+                    item = data;
+                }
+
+                // 转发的长微博处理
+                const retweet = item.retweeted_status;
+                if (retweet?.isLongText) {
+                    const retweetData = await ctx.cache.tryGet(`weibo:retweeted:${retweet.user.id}:${retweet.id}`, () => weiboUtils.getShowData(retweet.user.id, retweet.id));
+                    if (retweetData?.text) {
+                        item.retweeted_status.text = retweetData.text;
                     }
+                }
+
+                const link = `https://weibo.com/${uid}/${item.id}`;
+
+                const formatExtended = weiboUtils.formatExtended(ctx, item);
+                let description = formatExtended.description;
+                const title = formatExtended.title;
+                const pubDate = isDataOK ? parseDate(data.created_at) : parseDate(item.created_at);
 
-                    let title = '';
-                    if (res.length >= 2) {
-                        title = `${item.user.name}--转发微博`;
-                    } else if (item.pic_urls.length > 0) {
-                        title = `${item.user.name}--原创图文微博`;
+                // 视频的处理
+                if (displayVideo === '1') {
+                    // 含被转发微博时需要从被转发微博中获取视频
+                    if (item.retweeted_status) {
+                        description = weiboUtils.formatVideo(description, item.retweeted_status);
                     } else {
-                        title = `${item.user.name}--原创微博`;
+                        description = weiboUtils.formatVideo(description, item);
                     }
-                    return {
-                        title,
-                        content,
-                        author: item.user.name,
-                        date: item.created_at,
-                        link: `https://m.weibo.cn/detail/${item.id}`,
-                        pic_urls: picUrlList,
-                        videoList: Array.from(new Set(videoList)),
-                    };
-                });
-                return Promise.resolve(realContent);
-            })
-        );
+                }
 
-        ctx.state.data = {
-            title: '个人微博时间线',
-            link: 'https://weibo.com',
-            item: data.map((item) => {
-                let description = '';
-                if (item.content !== null) {
-                    description = item.content;
+                // 评论的处理
+                if (displayComments === '1') {
+                    description = await weiboUtils.formatComments(ctx, description, item);
                 }
 
-                item.pic_urls.forEach((pic) => {
-                    description += `<img alt="配图" style="margin: 0 auto" src="${pic.thumbnail_pic.replace('thumbnail', 'large')}"><br>`;
-                });
-                if (item.videoList.length !== 0) {
-                    item.videoList.forEach((url) => {
-                        description += `<iframe height=720 width=1080 src='${url}' frameborder=0 'allowfullscreen'></iframe>`;
-                    });
+                // 文章的处理
+                if (displayArticle === '1') {
+                    // 含被转发微博时需要从被转发微博中获取文章
+                    if (item.retweeted_status) {
+                        description = await weiboUtils.formatArticle(ctx, description, item.retweeted_status);
+                    } else {
+                        description = await weiboUtils.formatArticle(ctx, description, item);
+                    }
                 }
+
                 return {
-                    description: description
-                        .replace(/\b(url|href|src)\b=(['|"])http:/g, '$1=$2https:')
-                        .replace('http://t.cn/', 'https://t.cn/')
-                        .replace('http://m.weibo.', 'https://m.weibo.'),
-                    link: item.link,
-                    pubDate: item.date,
-                    title: item.title,
-                    author: item.author,
+                    title,
+                    description,
+                    link,
+                    pubDate,
+                    author: item.user.screen_name,
                 };
-            }),
+            })
+        );
+
+        ctx.state.data = {
+            title: `个人微博时间线--${name}`,
+            link: `http://weibo.com/${uid}/`,
+            description,
+            image: profileImageUrl,
+            item: resultItem,
         };
     } else if (uid === '0' || ctx.querystring) {
         const { app_key = '', redirect_url = ctx.request.origin + '/weibo/timeline/0', app_secret = '' } = config.weibo;
 
-        const code = Object.values(ctx.query)[0];
+        const code = ctx.query.code;
+        const routeParams = ctx.query.state;
         if (code) {
             const rep = await got.post(`https://api.weibo.com/oauth2/access_token?client_id=${app_key}&client_secret=${app_secret}&code=${code}&redirect_uri=${redirect_url}&grant_type=authorization_code`);
             const token = rep.data.access_token;
             const uid = rep.data.uid;
             const expires_in = rep.data.expires_in;
-            await ctx.cache.set('weibotimelineuid' + uid, token, expires_in, false);
+            await ctx.cache.set('weibotimelineuid' + uid, token, expires_in);
 
             ctx.set({
                 'Content-Type': 'text/html; charset=UTF-8',
                 'Cache-Control': 'no-cache',
             });
-            ctx.body = `<script>window.location = '/weibo/timeline/${uid}'</script>`;
+            ctx.body = `<script>window.location = '/weibo/timeline/${uid}${routeParams ? `/${routeParams}` : ''}'</script>`;
         }
     } else {
         const { app_key = '', redirect_url = ctx.request.origin + '/weibo/timeline/0' } = config.weibo;
@@ -173,6 +157,6 @@ module.exports = async (ctx) => {
         ctx.set({
             'Cache-Control': 'no-cache',
         });
-        ctx.redirect(`https://api.weibo.com/oauth2/authorize?client_id=${app_key}&redirect_uri=${redirect_url}`);
+        ctx.redirect(`https://api.weibo.com/oauth2/authorize?client_id=${app_key}&redirect_uri=${redirect_url}${routeParams ? `&state=${feature}/${routeParams.replace(/&/g, '%26')}` : ''}`);
     }
 };

package/lib/routes/weibo/user.js

@@ -1,6 +1,7 @@
 const querystring = require('querystring');
 const got = require('@/utils/got');
 const weiboUtils = require('./utils');
+const config = require('@/config').value;
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const { fallback, queryToBoolean } = require('@/utils/readable-social');
@@ -8,57 +9,104 @@ const { fallback, queryToBoolean } = require('@/utils/readable-social');
 module.exports = async (ctx) => {
     const uid = ctx.params.uid;
     let displayVideo = '1';
+    let displayArticle = '0';
+    let displayComments = '0';
     if (ctx.params.routeParams) {
         if (ctx.params.routeParams === '1' || ctx.params.routeParams === '0') {
             displayVideo = ctx.params.routeParams;
         } else {
             const routeParams = querystring.parse(ctx.params.routeParams);
             displayVideo = fallback(undefined, queryToBoolean(routeParams.displayVideo), true) ? '1' : '0';
+            displayArticle = fallback(undefined, queryToBoolean(routeParams.displayArticle), false) ? '1' : '0';
+            displayComments = fallback(undefined, queryToBoolean(routeParams.displayComments), false) ? '1' : '0';
         }
     }
 
-    const { data: containerData } = await got({
-        method: 'get',
-        url: `https://m.weibo.cn/profile/info?uid=${uid}`,
-        headers: {
-            Referer: 'https://m.weibo.cn/',
+    const containerData = await ctx.cache.tryGet(
+        `weibo:user:index:${uid}`,
+        async () => {
+            const _r = await got({
+                method: 'get',
+                url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}`,
+                headers: {
+                    Referer: `https://m.weibo.cn/u/${uid}`,
+                    'MWeibo-Pwa': 1,
+                    'X-Requested-With': 'XMLHttpRequest',
+                },
+            });
+            return _r.data;
         },
-    });
-    const name = containerData.data.user.screen_name;
-    const description = containerData.data.user.description;
-    const profileImageUrl = containerData.data.user.profile_image_url;
-    const containerid = containerData.data.more.match(/\d+/)[0];
-
-    const response = await got({
-        method: 'get',
-        url: `https://m.weibo.cn/api/container/getIndex?containerid=${containerid}`,
-        headers: {
-            Referer: `https://m.weibo.cn/u/${uid}`,
-            'MWeibo-Pwa': 1,
-            'X-Requested-With': 'XMLHttpRequest',
+        config.cache.routeExpire,
+        false
+    );
+    const name = containerData.data.userInfo.screen_name;
+    const description = containerData.data.userInfo.description;
+    const profileImageUrl = containerData.data.userInfo.profile_image_url;
+    const containerId = containerData.data.tabsInfo.tabs.filter((item) => item.tab_type === 'weibo')[0].containerid;
+
+    const cards = await ctx.cache.tryGet(
+        `weibo:user:cards:${uid}:${containerId}`,
+        async () => {
+            const _r = await got({
+                method: 'get',
+                url: `https://m.weibo.cn/api/container/getIndex?type=uid&value=${uid}&containerid=${containerId}`,
+                headers: {
+                    Referer: `https://m.weibo.cn/u/${uid}`,
+                    'MWeibo-Pwa': 1,
+                    'X-Requested-With': 'XMLHttpRequest',
+                },
+            });
+            return _r.data.data.cards;
         },
-    });
+        config.cache.routeExpire,
+        false
+    );
+
+    let earliestDate;
 
     const resultItem = await Promise.all(
-        response.data.data.cards
-            .filter((item) => item.mblog)
+        cards
+            .reverse() // reverse first to ensure the pinned card will be processed lastly
             .map(async (item) => {
+                if (!item.mblog && item.card_group && Array.isArray(item.card_group) && item.card_group.length > 0) {
+                    // w/o `mblog`, it may have a `card_group` containing only 1 card
+                    // only get the 1st card because we haven't observed that there can be multiple cards in it
+                    item = item.card_group[0];
+                }
+
+                if (!item.mblog) {
+                    return; // drop
+                }
+
                 const key = 'weibo:user:' + item.mblog.bid;
                 const data = await ctx.cache.tryGet(key, () => weiboUtils.getShowData(uid, item.mblog.bid));
 
                 if (data && data.text) {
                     item.mblog.text = data.text;
                     item.mblog.created_at = parseDate(data.created_at);
+                    item.mblog.pics = data.pics;
                     if (item.mblog.retweeted_status && data.retweeted_status) {
                         item.mblog.retweeted_status.created_at = data.retweeted_status.created_at;
                     }
                 } else {
                     item.mblog.created_at = timezone(item.mblog.created_at, +8);
                 }
+
+                // drop too old pinned weibo, otherwise preserve it
+                if (!item.mblog.title || item.mblog.title.text !== '置顶') {
+                    if (!earliestDate || (item.mblog.created_at && earliestDate > item.mblog.created_at)) {
+                        earliestDate = item.mblog.created_at;
+                    }
+                } else {
+                    if (earliestDate && item.mblog.created_at && earliestDate > item.mblog.created_at) {
+                        return;
+                    }
+                }
+
                 // 转发的长微博处理
                 const retweet = item.mblog.retweeted_status;
                 if (retweet && retweet.isLongText) {
-                    const retweetData = await weiboUtils.getShowData(retweet.user.id, retweet.bid);
+                    const retweetData = await ctx.cache.tryGet(`weibo:retweeted:${retweet.user.id}:${retweet.bid}`, () => weiboUtils.getShowData(retweet.user.id, retweet.bid));
                     if (retweetData !== undefined && retweetData.text) {
                         item.mblog.retweeted_status.text = retweetData.text;
                     }
@@ -67,13 +115,32 @@ module.exports = async (ctx) => {
                 const link = `https://weibo.com/${uid}/${item.mblog.bid}`;
                 const formatExtended = weiboUtils.formatExtended(ctx, item.mblog);
                 let description = formatExtended.description;
-                const formattedTitle = formatExtended.title;
-                const title = formattedTitle;
+                const title = formatExtended.title;
                 const pubDate = item.mblog.created_at;
 
                 // 视频的处理
                 if (displayVideo === '1') {
-                    description = weiboUtils.formatVideo(description, item.mblog);
+                    // 含被转发微博时需要从被转发微博中获取视频
+                    if (item.mblog.retweeted_status) {
+                        description = weiboUtils.formatVideo(description, item.mblog.retweeted_status);
+                    } else {
+                        description = weiboUtils.formatVideo(description, item.mblog);
+                    }
+                }
+
+                // 评论的处理
+                if (displayComments === '1') {
+                    description = await weiboUtils.formatComments(ctx, description, item.mblog);
+                }
+
+                // 文章的处理
+                if (displayArticle === '1') {
+                    // 含被转发微博时需要从被转发微博中获取文章
+                    if (item.mblog.retweeted_status) {
+                        description = await weiboUtils.formatArticle(ctx, description, item.mblog.retweeted_status);
+                    } else {
+                        description = await weiboUtils.formatArticle(ctx, description, item.mblog);
+                    }
                 }
 
                 return {
@@ -91,6 +158,6 @@ module.exports = async (ctx) => {
         link: `http://weibo.com/${uid}/`,
         description,
         image: profileImageUrl,
-        item: resultItem,
+        item: resultItem.filter((item) => item).reverse(),
     };
 };