rsshub 1.0.0-master.f8b1c8d → 1.0.0-master.f91e29d

package/lib/v2/theverge/index.js

@@ -0,0 +1,98 @@
+const got = require('@/utils/got');
+const parser = require('@/utils/rss-parser');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    let link;
+
+    if (ctx.params.hub) {
+        link = `https://www.theverge.com/${ctx.params.hub}/rss/index.xml`;
+    } else {
+        link = 'https://www.theverge.com/rss/index.xml';
+    }
+
+    const feed = await parser.parseURL(link);
+
+    const items = await Promise.all(
+        feed.items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+
+                const $ = cheerio.load(response.data);
+
+                const content = $('#content');
+                const body = $('.duet--article--article-body-component-container');
+
+                // 处理封面图片
+
+                const cover = $('meta[property="og:image"]');
+
+                if (cover.length > 0) {
+                    $(`<img src=${cover[0].attribs.content}>`).insertBefore(body[0].childNodes[0]);
+                }
+
+                // 处理封面视频
+                $('div.l-col__main > div.c-video-embed, div.c-entry-hero > div.c-video-embed').each((i, e) => {
+                    const src = `https://volume.vox-cdn.com/embed/${e.attribs['data-volume-uuid']}?autoplay=false`;
+
+                    $(`<iframe src="${src}" style="border: 0; top: 0; left: 0; width: 100%; height: 100%; position: absolute;" allowfullscreen scrolling="no"></iframe>`).insertBefore(body[0].childNodes[0]);
+                });
+
+                // 处理封面视频
+                $('div.l-col__main > div.c-video-embed--media iframe').each((i, e) => {
+                    $(e).insertBefore(body[0].childNodes[0]);
+                });
+
+                // 处理文章图片
+                content.find('figure.e-image').each((i, e) => {
+                    let src, caption;
+
+                    // 处理 jpeg, png
+                    if ($(e).find('picture > source').length > 0) {
+                        src = $(e)
+                            .find('picture > img')[0]
+                            .attribs.srcset.match(/(?<=320w,).*?(?=520w)/g)[0]
+                            .trim();
+                    } else if ($(e).find('img.c-dynamic-image').length > 0) {
+                        // 处理 gif
+                        src = $(e).find('span.e-image__image')[0].attribs['data-original'];
+                    }
+
+                    // 处理 caption
+                    if ($(e).find('span.e-image__meta').length > 0) {
+                        caption = $(e).find('span.e-image__meta').text();
+                    }
+
+                    const figure = `<figure><img src=${src}>${caption ? `<br><figcaption>${caption}</figcaption>` : ''}</figure>`;
+
+                    $(figure).insertBefore(e);
+
+                    $(e).remove();
+                });
+
+                const lede = $('.duet--article--lede h2:first');
+                if (lede[0]) {
+                    lede.insertBefore(body[0].childNodes[0]);
+                }
+
+                // 移除无用 DOM
+                content.find('.duet--article--comments-join-the-conversation').remove();
+                content.find('.duet--recirculation--related-list').remove();
+                delete item.content;
+                delete item.contentSnippet;
+                delete item.isoDate;
+
+                item.description = body.html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: items,
+    };
+};

package/lib/v2/theverge/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:hub?': ['HenryQW', 'vbali'],
+};

package/lib/v2/theverge/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'theverge.com': {
+        _name: 'The Verge',
+        '.': [
+            {
+                title: 'The Verge',
+                docs: 'https://docs.rsshub.app/new-media.html#the-verge',
+                source: ['/:hub', '/'],
+                target: '/theverge/:hub?',
+            },
+        ],
+    },
+};

package/lib/v2/theverge/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:hub?', require('./index'));
+};

package/lib/v2/tingshuitz/changsha.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const baseUrl = 'http://www.supplywater.com';
+
+module.exports = async (ctx) => {
+    const { channelId = 78 } = ctx.params;
+    const listPage = await got('http://www.supplywater.com/tstz-' + channelId + '.aspx');
+    const $ = cheerio.load(listPage.data);
+    const pageName = $('.mainRightBox .news-title').text();
+    const list = $('.mainRightBox .announcements-title a')
+        .map((_, item) => {
+            item = $(item);
+
+            return {
+                title: item.text().trim(),
+                link: baseUrl + item.attr('href').trim(),
+            };
+        })
+        .get();
+
+    const items = await Promise.all(
+        list.map(async (item) => {
+            const postPage = await got(item.link);
+            const $ = cheerio.load(postPage.data);
+
+            const data = {
+                title: item.title,
+                description: $('.mainRightBox div:last').html().trim(),
+                pubDate: parseDate($('.mainRightBox .gxsj span:first').text() + ' +0800', 'YYYY/M/D H:m:s ZZ'),
+                link: item.link,
+                author: $('.mainRightBox .gxsj span:last').text(),
+            };
+            return data;
+        })
+    );
+
+    ctx.state.data = {
+        title: `${pageName}通知 - 长沙水业集团`,
+        link: `${baseUrl}/fuwuzhinan.aspx`,
+        item: items,
+    };
+};

package/lib/v2/tingshuitz/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/changsha/:channelId?': ['shansing'],
     '/dalian': ['DIYgod'],
     '/dongguan': ['victoriqueko'],
     '/guangzhou': ['xyqfer'],

package/lib/v2/tingshuitz/radar.js

@@ -32,6 +32,16 @@ module.exports = {
             },
         ],
     },
+    'supplywater.com': {
+        _name: '停水通知',
+        www: [
+            {
+                title: '长沙市',
+                docs: 'https://docs.rsshub.app/forecast.html#ting-shui-tong-zhi',
+                source: ['/*'],
+            },
+        ],
+    },
     'whwater.com': {
         _name: '停水通知',
         '.': [

package/lib/v2/tingshuitz/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
+    router.get('/changsha/:channelId?', require('./changsha'));
     router.get('/dalian', require('./dalian'));
     router.get('/dongguan', require('./dongguan'));
     router.get('/guangzhou', require('./guangzhou'));

package/lib/v2/tingshuitz/xiaoshan.js

@@ -3,7 +3,7 @@ const cheerio = require('cheerio');
 
 module.exports = async (ctx) => {
     // const area = ctx.params.area;
-    const url = 'http://www.xswater.com/gongshui/channels/227.html';
+    const url = 'https://www.xswater.com/gongshui/channels/227.html';
     const response = await got({
         method: 'get',
         url,
@@ -15,7 +15,7 @@ module.exports = async (ctx) => {
 
     ctx.state.data = {
         title: $('title').text(),
-        link: 'http://www.xswater.com/gongshui/channels/227.html',
+        link: 'https://www.xswater.com/gongshui/channels/227.html',
         description: $('meta[name="description"]').attr('content') || $('title').text(),
         item:
             list &&
@@ -26,7 +26,7 @@ module.exports = async (ctx) => {
                         title: item.find('a').text(),
                         description: `萧山区停水通知：${item.find('a').text()}`,
                         pubDate: new Date(item.find('span').text().slice(1, 11)).toUTCString(),
-                        link: `http://www.xswater.com${item.find('a').attr('href')}`,
+                        link: `https://www.xswater.com${item.find('a').attr('href')}`,
                     };
                 })
                 .get(),

package/lib/v2/twitter/utils.js

@@ -28,12 +28,13 @@ const getOriginalImg = (url) => {
     }
 };
 const replaceBreak = (text) => text.replace(/<br><br>|<br>/g, ' ');
-const formatText = (text) =>
-    text
-        .replace(/https:\/\/t\.co(.*)/g, '')
-        .trim()
-        .replace(/\n/g, '<br>');
-const formatTextToPlain = (text) => text.replace(/https:\/\/t\.co(.*)/g, '').trim();
+
+const formatText = (text, urls) => {
+    urls.forEach((url) => {
+        text = text.replaceAll(url.url, url.expanded_url);
+    });
+    return text.trim().replace(/\n/g, '<br>');
+};
 
 const ProcessFeed = (ctx, { data = [] }, params = {}) => {
     // undefined and strings like "exclude_rts_replies" is also safely parsed, so no if branch is needed
@@ -47,6 +48,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
         showQuotedAuthorAvatarInDesc: fallback(params.showQuotedAuthorAvatarInDesc, queryToBoolean(routeParams.get('showQuotedAuthorAvatarInDesc')), false),
         showAuthorAvatarInDesc: fallback(params.showAuthorAvatarInDesc, queryToBoolean(routeParams.get('showAuthorAvatarInDesc')), false),
         showEmojiForRetweetAndReply: fallback(params.showEmojiForRetweetAndReply, queryToBoolean(routeParams.get('showEmojiForRetweetAndReply')), false),
+        showSymbolForRetweetAndReply: fallback(params.showSymbolForRetweetAndReply, queryToBoolean(routeParams.get('showSymbolForRetweetAndReply')), true),
         showRetweetTextInTitle: fallback(params.showRetweetTextInTitle, queryToBoolean(routeParams.get('showRetweetTextInTitle')), true),
         addLinkForPics: fallback(params.addLinkForPics, queryToBoolean(routeParams.get('addLinkForPics')), false),
         showTimestampInDescription: fallback(params.showTimestampInDescription, queryToBoolean(routeParams.get('showTimestampInDescription')), false),
@@ -68,6 +70,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
         showQuotedAuthorAvatarInDesc,
         showAuthorAvatarInDesc,
         showEmojiForRetweetAndReply,
+        showSymbolForRetweetAndReply,
         showRetweetTextInTitle,
         addLinkForPics,
         showTimestampInDescription,
@@ -172,29 +175,13 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
         return picsPrefix;
     };
 
-    const formatUrl = (item) => {
-        let url = '';
-        item.entities.urls &&
-            item.entities.urls.forEach((u) => {
-                if (readable) {
-                    url += '<br>';
-                } else {
-                    url += '&ensp;';
-                }
-                url += `<a href='${u.expanded_url}' target='_blank' rel='noopener noreferrer'>${u.expanded_url}</a>`;
-            });
-
-        return url;
-    };
-
     return data.map((item) => {
         const originalItem = item;
         item = item.retweeted_status || item;
         item.full_text = item.full_text || item.text;
-        item.full_text = formatText(item.full_text);
+        item.full_text = formatText(item.full_text, item.entities.urls);
         const img = formatMedia(item);
         let picsPrefix = generatePicsPrefix(item);
-        let url = '';
         let quote = '';
         let quoteInTitle = '';
 
@@ -205,6 +192,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
             if (quoteData) {
                 quoteData.full_text = quoteData.full_text || quoteData.text;
                 const author = quoteData.user;
+                quote += '<div class="rsshub-quote">';
                 if (readable) {
                     quote += `<br clear='both' /><div style='clear: both'></div>`;
                     quote += `<blockquote style='background: #80808010;border-top:1px solid #80808030;border-bottom:1px solid #80808030;margin:0;padding:5px 20px;'>`;
@@ -235,16 +223,15 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
                 }
 
                 quote += `:&ensp;`;
-                quote += formatText(quoteData.full_text);
+                quote += formatText(quoteData.full_text, quoteData.entities.urls);
 
                 if (!readable) {
                     quote += '<br>';
                 }
                 quote += formatMedia(quoteData);
                 picsPrefix += generatePicsPrefix(quoteData);
-                quote += formatUrl(quoteData);
-                quoteInTitle += showEmojiForRetweetAndReply ? ' 💬 ' : ' RT ';
-                quoteInTitle += `${author.name}: ${formatTextToPlain(quoteData.full_text)}`;
+                quoteInTitle += showEmojiForRetweetAndReply ? ' 💬 ' : showSymbolForRetweetAndReply ? ' RT ' : '';
+                quoteInTitle += `${author.name}: ${formatText(quoteData.full_text, quoteData.entities.urls)}`;
 
                 if (readable) {
                     quote += `<br><small>Link: <a href='https://twitter.com/${author.screen_name}/status/${quoteData.id_str}' target='_blank' rel='noopener noreferrer'>https://twitter.com/${author.screen_name}/status/${quoteData.id_str}</a></small>`;
@@ -260,11 +247,8 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
                 if (readable) {
                     quote += `</blockquote>`;
                 }
-            } else {
-                url = formatUrl(item);
+                quote += '</div>';
             }
-        } else {
-            url = formatUrl(item);
         }
 
         // Make title
@@ -276,15 +260,15 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
         const isQuote = item.is_quote_status;
         if (!isRetweet && (!isQuote || showRetweetTextInTitle)) {
             if (item.in_reply_to_screen_name) {
-                title += showEmojiForRetweetAndReply ? '↩️ ' : 'Re ';
+                title += showEmojiForRetweetAndReply ? '↩️ ' : showSymbolForRetweetAndReply ? 'Re ' : '';
             }
             title += replaceBreak(originalItem.full_text);
         }
         if (isRetweet) {
-            title += showEmojiForRetweetAndReply ? '🔁 ' : 'RT ';
+            title += showEmojiForRetweetAndReply ? '🔁 ' : showSymbolForRetweetAndReply ? 'RT ' : '';
             title += item.user.name + ': ';
             if (item.in_reply_to_screen_name) {
-                title += showEmojiForRetweetAndReply ? ' ↩️ ' : ' Re ';
+                title += showEmojiForRetweetAndReply ? ' ↩️ ' : showSymbolForRetweetAndReply ? ' Re ' : '';
             }
             title += replaceBreak(item.full_text);
         }
@@ -316,7 +300,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
                 }
                 description += '&ensp;';
             }
-            description += showEmojiForRetweetAndReply ? '🔁' : 'RT';
+            description += showEmojiForRetweetAndReply ? '🔁' : showSymbolForRetweetAndReply ? 'RT' : '';
             if (!showAuthorInDesc) {
                 description += '&ensp;';
                 if (readable) {
@@ -359,11 +343,10 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
             description += `:&ensp;`;
         }
         if (item.in_reply_to_screen_name) {
-            description += showEmojiForRetweetAndReply ? '↩️ ' : 'Re ';
+            description += showEmojiForRetweetAndReply ? '↩️ ' : showSymbolForRetweetAndReply ? 'Re ' : '';
         }
 
         description += item.full_text;
-        description += url;
         description += img;
         description += quote;
 
@@ -382,6 +365,32 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
             description,
             pubDate: parseDate(item.created_at),
             link: `https://twitter.com/${item.user.screen_name}/status/${item.id_str}`,
+
+            _extra:
+                (isRetweet && {
+                    links: [
+                        {
+                            type: 'repost',
+                        },
+                    ],
+                }) ||
+                (item.is_quote_status && {
+                    links: [
+                        {
+                            url: `https://twitter.com/${item.quoted_status?.user?.screen_name}/status/${item.quoted_status?.id_str}`,
+                            type: 'quote',
+                        },
+                    ],
+                }) ||
+                (item.in_reply_to_screen_name &&
+                    item.in_reply_to_status_id_str && {
+                        links: [
+                            {
+                                url: `https://twitter.com/${item.in_reply_to_screen_name}/status/${item.in_reply_to_status_id_str}`,
+                                type: 'reply',
+                            },
+                        ],
+                    }),
         };
     });
 };

package/lib/v2/twitter/web-api/twitter-api.js

@@ -115,24 +115,28 @@ function gatherLegacyFromData(entries, filter = 'tweet-') {
     const tweets = [];
     entries.forEach((entry) => {
         if (entry.entryId && entry.entryId.includes(filter)) {
-            const tweet = entry.content.itemContent.tweet_results.result;
-            const retweet = tweet.legacy.retweeted_status_result?.result;
-            for (const t of [tweet, retweet]) {
-                if (!t?.legacy) {
-                    continue;
+            const tweet = entry.content.itemContent.tweet_results?.result;
+            if (tweet) {
+                const retweet = tweet.legacy?.retweeted_status_result?.result;
+                for (const t of [tweet, retweet]) {
+                    if (!t?.legacy) {
+                        continue;
+                    }
+                    t.legacy.user = t.core.user_results.result.legacy;
+                    const quote = t.quoted_status_result?.result;
+                    if (quote) {
+                        t.legacy.quoted_status = quote.legacy;
+                        t.legacy.quoted_status.user = quote.core.user_results.result.legacy;
+                    }
                 }
-                t.legacy.user = t.core.user_results.result.legacy;
-                const quote = t.quoted_status_result?.result;
-                if (quote) {
-                    t.legacy.quoted_status = quote.legacy;
-                    t.legacy.quoted_status.user = quote.core.user_results.result.legacy;
+                const legacy = tweet.legacy;
+                if (legacy) {
+                    if (retweet) {
+                        legacy.retweeted_status = retweet.legacy;
+                    }
+                    tweets.push(legacy);
                 }
             }
-            const legacy = tweet.legacy;
-            if (retweet) {
-                legacy.retweeted_status = retweet.legacy;
-            }
-            tweets.push(legacy);
         }
     });
     return tweets;
@@ -195,13 +199,13 @@ const cacheTryGet = async (cache, screenName, params, func) => {
     const id = await getUserID(cache, screenName);
     const funcName = func.name;
     const paramsString = JSON.stringify(params);
-    return await cache.tryGet(`twitter:${id}:${funcName}:${paramsString}`, async () => await func(id, params), config.cache.routeExpire, false);
+    return cache.tryGet(`twitter:${id}:${funcName}:${paramsString}`, () => func(id, params), config.cache.routeExpire, false);
 };
 
-const getUserTweets = async (cache, screenName, params = {}) => await cacheTryGet(cache, screenName, params, getUserTweetsByID);
-const getUserTweetsAndReplies = async (cache, screenName, params = {}) => await cacheTryGet(cache, screenName, params, getUserTweetsAndRepliesByID);
-const getUserMedia = async (cache, screenName, params = {}) => await cacheTryGet(cache, screenName, params, getUserMediaByID);
-const getUserLikes = async (cache, screenName, params = {}) => await cacheTryGet(cache, screenName, params, getUserLikesByID);
+const getUserTweets = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserTweetsByID);
+const getUserTweetsAndReplies = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserTweetsAndRepliesByID);
+const getUserMedia = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserMediaByID);
+const getUserLikes = (cache, screenName, params = {}) => cacheTryGet(cache, screenName, params, getUserLikesByID);
 
 const getSearch = async (keywords, params = {}) => gatherLegacyFromLegacyApiData(await timelineKeywords(keywords, params), 'sq-I-t-');
 

package/lib/v2/twreporter/category.js

@@ -8,16 +8,25 @@ module.exports = async (ctx) => {
     const url = baseURL + `/categories/${ctx.params.cid}`;
     const res = await got(url);
     const $ = cheerio.load(res.data);
-    const list = $('.lnKPLr').get();
-    const category = $('.kCfkTU').text();
+    const regexp = /^window\.__REDUX_STATE__=(.*);$/gm;
+    const raw = $('script[charset="UTF-8"]').html().replace(regexp, '$1');
+    const list = JSON.parse(raw);
+    const ids = list.entities.posts.allIds;
+
+    const links = [];
+    const titles = [];
+    const category = list.entities.posts.byId[ids[0]].categories[0].name;
+
+    for (const id of ids) {
+        links.push(baseURL + '/a/' + list.entities.posts.byId[id].slug);
+        titles.push(list.entities.posts.byId[id].title);
+    }
 
     const out = await Promise.all(
-        list.map((item) => {
-            const $ = cheerio.load(item);
-            const address = baseURL + $('a').attr('href');
-            const title = $('.list-item__Title-sc-1dx5lew-5').text();
-            return ctx.cache.tryGet(address, async () => {
-                const single = await fetch(address);
+        links.map((item, index) => {
+            const title = titles[index];
+            return ctx.cache.tryGet(item, async () => {
+                const single = await fetch(item);
                 single.title = title;
                 return single;
             });

package/lib/v2/typora/changelog-dev.js

@@ -0,0 +1,31 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const currentUrl = 'https://typora.io/releases/dev';
+    const response = await got(currentUrl);
+
+    const $ = cheerio.load(response.data);
+
+    const items = $('h2')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: `${currentUrl}#${item.text()}`,
+                description: item
+                    .nextUntil('h2')
+                    .toArray()
+                    .map((item) => $(item).html())
+                    .join(''),
+            };
+        });
+
+    ctx.state.data = {
+        title: `Typora Changelog - Dev`,
+        link: currentUrl,
+        description: 'Typora Changelog',
+        item: items,
+    };
+};

package/lib/v2/typora/changelog.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const host = 'https://support.typora.io';
+
+    const { data } = await got(`${host}/store/`);
+
+    const list = Object.values(data)
+        .filter((i) => i.category === 'new')
+        .map((i) => ({
+            title: i.title,
+            author: i.author,
+            description: i.content,
+            link: `${host}${i.url}`,
+        }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+
+                item.pubDate = parseDate($('.post-meta time').text());
+                item.description = $('#post-content').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'Typora Changelog',
+        link: host,
+        description: 'Typora Changelog',
+        image: `${host}/assets/img/favicon-128.png`,
+        item: items,
+    };
+};

package/lib/v2/typora/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/changelog': ['cnzgray'],
+    '/changelog/dev': ['nczitzk'],
+};

package/lib/v2/typora/radar.js

@@ -0,0 +1,21 @@
+module.exports = {
+    'typora.io': {
+        _name: 'Typora',
+        '.': [
+            {
+                title: 'Dev Release Changelog',
+                docs: 'https://docs.rsshub.app/program-update.html#typora',
+                source: ['/releases/dev'],
+                target: '/typora/changelog/dev',
+            },
+        ],
+        support: [
+            {
+                title: 'Changelog',
+                docs: 'https://docs.rsshub.app/program-update.html#typora',
+                source: ['/'],
+                target: '/typora/changelog',
+            },
+        ],
+    },
+};

package/lib/v2/typora/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/changelog', require('./changelog'));
+    router.get('/changelog/dev', require('./changelog-dev'));
+};