rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f796ff1

package/lib/v2/theverge/index.js

@@ -0,0 +1,98 @@
+const got = require('@/utils/got');
+const parser = require('@/utils/rss-parser');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    let link;
+
+    if (ctx.params.hub) {
+        link = `https://www.theverge.com/${ctx.params.hub}/rss/index.xml`;
+    } else {
+        link = 'https://www.theverge.com/rss/index.xml';
+    }
+
+    const feed = await parser.parseURL(link);
+
+    const items = await Promise.all(
+        feed.items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+
+                const $ = cheerio.load(response.data);
+
+                const content = $('#content');
+                const body = $('.duet--article--article-body-component-container');
+
+                // 处理封面图片
+
+                const cover = $('meta[property="og:image"]');
+
+                if (cover.length > 0) {
+                    $(`<img src=${cover[0].attribs.content}>`).insertBefore(body[0].childNodes[0]);
+                }
+
+                // 处理封面视频
+                $('div.l-col__main > div.c-video-embed, div.c-entry-hero > div.c-video-embed').each((i, e) => {
+                    const src = `https://volume.vox-cdn.com/embed/${e.attribs['data-volume-uuid']}?autoplay=false`;
+
+                    $(`<iframe src="${src}" style="border: 0; top: 0; left: 0; width: 100%; height: 100%; position: absolute;" allowfullscreen scrolling="no"></iframe>`).insertBefore(body[0].childNodes[0]);
+                });
+
+                // 处理封面视频
+                $('div.l-col__main > div.c-video-embed--media iframe').each((i, e) => {
+                    $(e).insertBefore(body[0].childNodes[0]);
+                });
+
+                // 处理文章图片
+                content.find('figure.e-image').each((i, e) => {
+                    let src, caption;
+
+                    // 处理 jpeg, png
+                    if ($(e).find('picture > source').length > 0) {
+                        src = $(e)
+                            .find('picture > img')[0]
+                            .attribs.srcset.match(/(?<=320w,).*?(?=520w)/g)[0]
+                            .trim();
+                    } else if ($(e).find('img.c-dynamic-image').length > 0) {
+                        // 处理 gif
+                        src = $(e).find('span.e-image__image')[0].attribs['data-original'];
+                    }
+
+                    // 处理 caption
+                    if ($(e).find('span.e-image__meta').length > 0) {
+                        caption = $(e).find('span.e-image__meta').text();
+                    }
+
+                    const figure = `<figure><img src=${src}>${caption ? `<br><figcaption>${caption}</figcaption>` : ''}</figure>`;
+
+                    $(figure).insertBefore(e);
+
+                    $(e).remove();
+                });
+
+                const lede = $('.duet--article--lede h2:first');
+                if (lede[0]) {
+                    lede.insertBefore(body[0].childNodes[0]);
+                }
+
+                // 移除无用 DOM
+                content.find('.duet--article--comments-join-the-conversation').remove();
+                content.find('.duet--recirculation--related-list').remove();
+                delete item.content;
+                delete item.contentSnippet;
+                delete item.isoDate;
+
+                item.description = body.html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: items,
+    };
+};

package/lib/v2/theverge/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:hub?': ['HenryQW', 'vbali'],
+};

package/lib/v2/theverge/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'theverge.com': {
+        _name: 'The Verge',
+        '.': [
+            {
+                title: 'The Verge',
+                docs: 'https://docs.rsshub.app/new-media.html#the-verge',
+                source: ['/:hub', '/'],
+                target: '/theverge/:hub?',
+            },
+        ],
+    },
+};

package/lib/v2/theverge/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:hub?', require('./index'));
+};

package/lib/v2/thwiki/index.js

@@ -0,0 +1,32 @@
+const got = require('@/utils/got');
+const dayjs = require('dayjs');
+
+module.exports = async (ctx) => {
+    const beforeDays = ctx.params.before ? parseInt(ctx.params.before) : 30;
+    const afterDays = ctx.params.after ? parseInt(ctx.params.after) : 30;
+    const before = dayjs().subtract(beforeDays, 'day').format('YYYY-MM-DD');
+    const after = dayjs().add(afterDays, 'day').format('YYYY-MM-DD');
+
+    const response = await got({
+        method: 'get',
+        url: `https://calendar-serverless.thwiki.cc/api/events/${before}/${after}`,
+        headers: {
+            Origin: 'https://thwiki.cc',
+        },
+    });
+    const data = response.data.results;
+
+    ctx.state.data = {
+        title: 'Touhou events calendar (THBWiki)',
+        link: `https://calendar.thwiki.cc/`,
+        description: 'A Touhou related events calendar api from THBWiki',
+        item: data.map((item) => ({
+            title: item.title,
+            author: item.title,
+            category: item.type ? item.type[0] : '活动',
+            description: `${item.desc}. 开始时间: ${item.startStr}. 结束时间: ${item.endStr}.${item.type ? ' 活动类型: ' + item.type[0] : ''}`,
+            guid: item.id,
+            link: item.url,
+        })),
+    };
+};

package/lib/v2/thwiki/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/calendar/:before?/:after?': ['aether17'],
+};

package/lib/v2/thwiki/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'thwiki.cc': {
+        _name: 'THBWiki',
+        '.': [
+            {
+                title: '日历',
+                docs: 'https://docs.rsshub.app/#thbwiki',
+                source: ['/', '/日程表'],
+                target: '/thwiki/calendar',
+            },
+        ],
+    },
+};

package/lib/v2/thwiki/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/calendar/:before?/:after?', require('./index'));
+};

package/lib/v2/tiktok/user.js

@@ -18,7 +18,7 @@ module.exports = async (ctx) => {
                 request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
             });
             await page.goto(`${baseUrl}/${user}`, {
-                waitUntil: 'domcontentloaded',
+                waitUntil: 'networkidle0',
             });
             const SIGI_STATE = await page.evaluate(() => window.SIGI_STATE);
             browser.close();

package/lib/v2/tingshuitz/changsha.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const baseUrl = 'http://www.supplywater.com';
+
+module.exports = async (ctx) => {
+    const { channelId = 78 } = ctx.params;
+    const listPage = await got('http://www.supplywater.com/tstz-' + channelId + '.aspx');
+    const $ = cheerio.load(listPage.data);
+    const pageName = $('.mainRightBox .news-title').text();
+    const list = $('.mainRightBox .announcements-title a')
+        .map((_, item) => {
+            item = $(item);
+
+            return {
+                title: item.text().trim(),
+                link: baseUrl + item.attr('href').trim(),
+            };
+        })
+        .get();
+
+    const items = await Promise.all(
+        list.map(async (item) => {
+            const postPage = await got(item.link);
+            const $ = cheerio.load(postPage.data);
+
+            const data = {
+                title: item.title,
+                description: $('.mainRightBox div:last').html().trim(),
+                pubDate: parseDate($('.mainRightBox .gxsj span:first').text() + ' +0800', 'YYYY/M/D H:m:s ZZ'),
+                link: item.link,
+                author: $('.mainRightBox .gxsj span:last').text(),
+            };
+            return data;
+        })
+    );
+
+    ctx.state.data = {
+        title: `${pageName}通知 - 长沙水业集团`,
+        link: `${baseUrl}/fuwuzhinan.aspx`,
+        item: items,
+    };
+};

package/lib/v2/tingshuitz/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/changsha/:channelId?': ['shansing'],
     '/dalian': ['DIYgod'],
     '/dongguan': ['victoriqueko'],
     '/guangzhou': ['xyqfer'],

package/lib/v2/tingshuitz/radar.js

@@ -32,6 +32,16 @@ module.exports = {
             },
         ],
     },
+    'supplywater.com': {
+        _name: '停水通知',
+        www: [
+            {
+                title: '长沙市',
+                docs: 'https://docs.rsshub.app/forecast.html#ting-shui-tong-zhi',
+                source: ['/*'],
+            },
+        ],
+    },
     'whwater.com': {
         _name: '停水通知',
         '.': [

package/lib/v2/tingshuitz/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
+    router.get('/changsha/:channelId?', require('./changsha'));
     router.get('/dalian', require('./dalian'));
     router.get('/dongguan', require('./dongguan'));
     router.get('/guangzhou', require('./guangzhou'));

package/lib/v2/trow/portal.js

@@ -4,11 +4,21 @@ const { parseDate } = require('@/utils/parse-date');
 const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
+    let data;
+    const response = await got.extend({ followRedirect: false }).get({
         url: `https://trow.cc`,
     });
-    const data = response.data;
+    if (response.statusCode === 302) {
+        const response2 = await got.extend({ followRedirect: false }).get({
+            url: `https://trow.cc`,
+            headers: {
+                cookie: response.headers['set-cookie'],
+            },
+        });
+        data = response2.data;
+    } else {
+        data = response.data;
+    }
 
     const $ = cheerio.load(data);
     const list = $('#portal_content .borderwrap[style="display:show"]');

package/lib/v2/twitter/web-api/twitter-got.js

@@ -6,8 +6,8 @@ const config = require('@/config').value;
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L716-L726
 const headers = {
     authorization: config.twitter.authorization,
-    // every single user and guest share the same token:
-    // Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA
+    // Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw
+    // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25
     'x-guest-token': undefined,
     'x-twitter-auth-type': undefined,
     'x-twitter-client-language': 'en',

package/lib/v2/twreporter/category.js

@@ -8,16 +8,25 @@ module.exports = async (ctx) => {
     const url = baseURL + `/categories/${ctx.params.cid}`;
     const res = await got(url);
     const $ = cheerio.load(res.data);
-    const list = $('.lnKPLr').get();
-    const category = $('.kCfkTU').text();
+    const regexp = /^window\.__REDUX_STATE__=(.*);$/gm;
+    const raw = $('script[charset="UTF-8"]').html().replace(regexp, '$1');
+    const list = JSON.parse(raw);
+    const ids = list.entities.posts.allIds;
+
+    const links = [];
+    const titles = [];
+    const category = list.entities.posts.byId[ids[0]].categories[0].name;
+
+    for (const id of ids) {
+        links.push(baseURL + '/a/' + list.entities.posts.byId[id].slug);
+        titles.push(list.entities.posts.byId[id].title);
+    }
 
     const out = await Promise.all(
-        list.map((item) => {
-            const $ = cheerio.load(item);
-            const address = baseURL + $('a').attr('href');
-            const title = $('.list-item__Title-sc-1dx5lew-5').text();
-            return ctx.cache.tryGet(address, async () => {
-                const single = await fetch(address);
+        links.map((item, index) => {
+            const title = titles[index];
+            return ctx.cache.tryGet(item, async () => {
+                const single = await fetch(item);
                 single.title = title;
                 return single;
             });

package/lib/v2/uber/blog.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const rootURL = 'https://www.uber.com';
+const apiURL = 'https://blogapi.uber.com';
+
+module.exports = async (ctx) => {
+    let maxPage = Number(ctx.params.maxPage);
+    if (Number.isNaN(maxPage)) {
+        maxPage = 1;
+    }
+
+    let pages = await Promise.all(
+        [...Array(maxPage).keys()].map((pageIdx) =>
+            got(`${apiURL}/wp-json/blog/v1/data`, {
+                searchParams: {
+                    page: pageIdx + 1,
+                    parent: 'pittsburgh',
+                    slug: 'engineering',
+                },
+            })
+        )
+    );
+    pages = pages.map((page) => page.data);
+
+    const result = await Promise.all(
+        pages.map(
+            async (page) =>
+                await Promise.all(
+                    page.posts.map((post) =>
+                        ctx.cache.tryGet(`${rootURL}${post.link}`, async () => {
+                            let { data: article } = await got(`${apiURL}/wp-json/blog/v1/data`, {
+                                searchParams: {
+                                    slug: post.link.replace(/\//g, '').replace('blog', ''),
+                                },
+                            });
+                            article = article.article;
+
+                            return {
+                                link: article.link,
+                                title: article.title,
+                                description: article.content,
+                                pubDate: parseDate(article.created),
+                                author: article.author,
+                                category: article.categories.map((category) => category.category_name),
+                            };
+                        })
+                    )
+                )
+        )
+    );
+
+    ctx.state.data = {
+        title: `Uber Engineering Blog`,
+        link: rootURL + '/blog/engineering',
+        description: 'The technology behind Uber Engineering',
+        item: result.flat(),
+    };
+};

package/lib/v2/uber/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/blog/:maxPage?': ['hulb'],
+};

package/lib/v2/uber/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'uber.com': {
+        _name: 'Uber Blog',
+        www: [
+            {
+                title: 'Engineering',
+                docs: 'https://docs.rsshub.app/blog.html#you-bu',
+                source: ['/blog/pittsburgh/engineering'],
+                target: '/uber/blog',
+            },
+        ],
+    },
+};

package/lib/v2/uber/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/blog/:maxPage?', require('./blog'));
+};

package/lib/v2/udn/breaking-news.js

@@ -0,0 +1,85 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+    const link = `/news/breaknews/1/${id}#breaknews`;
+    const name = await getLinkName(ctx, link);
+    const rootUrl = 'https://udn.com';
+    const response = await got(`${rootUrl}/api/more?page=1&channelId=1&cate_id=${id}&type=breaknews`);
+    const items = await Promise.all(
+        response.data.lists.map((item) => {
+            let link = item.titleLink.indexOf('http://') === 0 || item.titleLink.indexOf('https://') === 0 ? item.titleLink : `${rootUrl}${item.titleLink}`;
+            const linkUrl = new URL(link);
+            // cleanup query paramter
+            linkUrl.query = linkUrl.search = '';
+            link = linkUrl.toString();
+            return ctx.cache.tryGet(link, async () => {
+                let result = await got(link);
+                // VIP article requires redirection
+                // e.g. https://udn.com/news/story/7331/6576320
+                const vip = result.data.match(/<script language=javascript>window\.location\.href="(https?:\/\/[^"]+")/);
+                if (vip) {
+                    result = await got(vip[1]);
+                }
+                const $ = cheerio.load(result.data);
+                const metadata = $('script[type="application/ld+json"]').eq(0).text().trim();
+                const data = metadata.startsWith('[') ? JSON.parse(metadata)[0] : JSON.parse(metadata);
+                // e.g. https://udn.com/news/story/7331/6576320
+                const content = $('.article-content__editor');
+                // e.g. https://udn.com/news/story/7240/6576424
+                const body = $('.article-body__editor');
+                let description = '';
+                if (data.image) {
+                    description += art(path.join(__dirname, 'templates/figure.art'), {
+                        src: data.image.contentUrl,
+                        alt: data.image.name,
+                    });
+                }
+                if (content.length) {
+                    description += content.html();
+                } else if (body.length) {
+                    description += body.html();
+                }
+                return {
+                    title: item.title,
+                    author: data.author.name,
+                    description,
+                    pubDate: timezone(parseDate(item.time.date, 'YYYY-MM-DD HH:mm'), +8),
+                    category: [data.articleSelection, ...data.keywords.split(',')],
+                    link,
+                };
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: `即時${name} - 聯合新聞網`,
+        link: `${rootUrl}${link}`,
+        description: 'udn.com 提供即時新聞以及豐富的政治、社會、地方、兩岸、國際、財經、數位、運動、NBA、娛樂、生活、健康、旅遊新聞，以最即時、多元的內容，滿足行動世代的需求',
+        item: items,
+    };
+};
+
+const getLinkName = async (ctx, link) => {
+    const url = 'https://udn.com/news/breaknews';
+    const links = await ctx.cache.tryGet(url, async () => {
+        const result = await got(url);
+        const $ = cheerio.load(result.data);
+        const data = $('.cate-list__subheader a')
+            .get()
+            .map((item) => {
+                item = $(item);
+                return { [item.attr('href')]: item.text().trim() };
+            });
+        return Object.assign({}, ...data);
+    });
+    if (link in links) {
+        return links[link];
+    }
+    return '列表';
+};

package/lib/v2/udn/global/index.js

@@ -0,0 +1,67 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? '';
+
+    const start = category === 'hot' ? 6 : 0;
+    const end = category === 'new' ? 6 : 12;
+
+    const rootUrl = 'https://global.udn.com';
+    const currentUrl = `${rootUrl}/global_vision/index${category ? `/${category}` : ''}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    $('.topic').remove();
+
+    let items = $('.news_cards ul li a')
+        .toArray()
+        .slice(start, end)
+        .concat(category === '' ? $('.last24, h2').find('a').toArray() : [])
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href');
+
+            return {
+                title: item.find('h3').text() || item.text(),
+                link: /^http/.test(link) ? link : `${rootUrl}${item.attr('href')}`,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('#story_art_title, #story_bady_info, #story_also').remove();
+                content('.social_bar, .photo_pop, .only_mobile, .area').remove();
+
+                item.description = content('#tags').prev().html();
+                item.author = content('#story_author_name').text();
+                item.pubDate = timezone(parseDate(content('meta[name="date"]').attr('content')), +8);
+                item.category = content('meta[name="news_keywords"]').attr('content').split(',');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/udn/global/tag.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const tag = ctx.params.tag ?? '過去24小時';
+
+    const rootUrl = 'https://global.udn.com';
+    const currentUrl = `${rootUrl}/search/tagging/1020/${tag}`;
+    const apiUrl = `${rootUrl}/search/ajax_tag/1020/${tag}`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let items = response.data.articles.map((item) => ({
+        title: item.TITLE,
+        author: item.AUTHOR,
+        pubDate: parseDate(item.TIMESTAMP * 1000),
+        category: item.TAG.map((t) => t.tag),
+        link: `${rootUrl}/global_vision/story/${item.CATE_ID}/${item.ART_ID}`,
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('#story_art_title, #story_bady_info, #story_also').remove();
+                content('.social_bar, .photo_pop, .only_mobile, .area').remove();
+
+                item.author = content('#story_author_name').text();
+                item.description = content('#tags').prev().html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `轉角國際 udn Global - ${tag}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/udn/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/global/:category?': ['nczitzk'],
+    '/global/tag/:tag?': ['emdoe', 'nczitzk'],
+    '/news/breakingnews/:id': ['miles170'],
+};

package/lib/v2/udn/radar.js

@@ -0,0 +1,27 @@
+module.exports = {
+    'udn.com': {
+        _name: '聯合新聞網',
+        '.': [
+            {
+                title: '即時新聞',
+                docs: 'https://docs.rsshub.app/new-media.html#lian-he-xin-wen-wang-ji-shi-xin-wen',
+                source: ['/news/breaknews/1/:id', '/'],
+                target: '/udn/news/breakingnews/:id',
+            },
+        ],
+        global: [
+            {
+                title: '轉角國際 - 首頁',
+                docs: 'https://docs.rsshub.app/new-media.html#lian-he-xin-wen-wang-zhuan-jiao-guo-ji-shou-ye',
+                source: ['/global_vision/index/:category', '/'],
+                target: '/udn/global/:category?',
+            },
+            {
+                title: '轉角國際 - 標籤',
+                docs: 'https://docs.rsshub.app/new-media.html#lian-he-xin-wen-wang-zhuan-jiao-guo-ji-biao-qian',
+                source: ['/search/tagging/1020/:tag', '/'],
+                target: '/udn/global/tag/:tag?',
+            },
+        ],
+    },
+};