rsshub 1.0.0-master.fc5a2f6 → 1.0.0-master.fc648b5

package/lib/v2/springer/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/journal/:journal', require('./journal'));
+};

package/lib/v2/springer/templates/description.art

@@ -0,0 +1,12 @@
+<p>
+    <span><big>{{ item.title }}</big></span><br>
+</p>
+<p>
+    <span><small><i>{{ item.authors }}</i></small></span><br>
+    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br>
+    <span><small><i>{{ item.issue }}</i></small></span><br>
+    <img src={{ item.img }}>
+</p>
+<p>
+    <span>{{ item.abstract }}</span><br>    
+</p>

package/lib/v2/supchina/index.js

@@ -0,0 +1,67 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://supchina.com';
+    const currentUrl = `${rootUrl}/feed/`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('item')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                guid: item.find('guid').text(),
+                title: item.find('title').text(),
+                link: item.find('guid').text(),
+                author: item
+                    .find('dc\\:creator')
+                    .html()
+                    .match(/CDATA\[(.*?)\]/)[1],
+                category: item
+                    .find('category')
+                    .toArray()
+                    .map(
+                        (c) =>
+                            $(c)
+                                .html()
+                                .match(/CDATA\[(.*?)\]/)[1]
+                    ),
+                pubDate: parseDate(item.find('pubDate').text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('.aspect-spacer, .post-recs, .author-bio').remove();
+
+                item.description = content('.post__main').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').first().text(),
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/supchina/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/podcasts': ['nczitzk'],
+    '/': ['nczitzk'],
+};

package/lib/v2/supchina/podcasts.js

@@ -0,0 +1,66 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://supchina.com';
+    const currentUrl = `${rootUrl}/feed/podcast`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('item')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                link: item.find('guid').text(),
+                author: item.find('itunes\\:author').text(),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const episodeResponse = await got({
+                    method: 'get',
+                    url: `https://rss.art19.com/episodes/${detailResponse.data.match(/data-episode-id="(.*?)"/)[1]}`,
+                    headers: {
+                        accept: 'application/json',
+                    },
+                });
+
+                const data = episodeResponse.data;
+
+                item.title = data.content.episode_title;
+                item.itunes_item_image = data.content.cover_image;
+                item.itunes_duration = data.content.duration;
+                item.enclosure_url = data.content.media.mp3.url;
+                item.enclosure_type = data.content.media.mp3.content_type;
+                item.description = data.content.episode_description;
+                item.pubDate = parseDate(data.performed_at);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'SupChina - Podcasts',
+        link: `${rootUrl}/podcasts`,
+        itunes_author: $('channel itunes\\:author').first().text(),
+        image: $('itunes\\:image').attr('href'),
+        item: items,
+    };
+};

package/lib/v2/supchina/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'supchina.com': {
+        _name: 'SupChina',
+        '.': [
+            {
+                title: 'Feed',
+                docs: 'https://docs.rsshub.app/new-media.html#supchina-feed',
+                source: ['/feed', '/'],
+                target: '/supchina',
+            },
+            {
+                title: 'Podcasts',
+                docs: 'https://docs.rsshub.app/new-media.html#supchina-podcasts',
+                source: ['/podcasts', '/'],
+                target: '/supchina/podcasts',
+            },
+        ],
+    },
+};

package/lib/v2/supchina/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/podcasts', require('./podcasts'));
+    router.get('/', require('./index'));
+};

package/lib/v2/techflow520/index.js

@@ -0,0 +1,34 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? '头条';
+
+    const rootUrl = 'https://www.techflow520.com';
+    const apiUrl = `${rootUrl}/index/index/tab_centet`;
+
+    const response = await got({
+        method: 'post',
+        url: apiUrl,
+        form: {
+            offect: 0,
+            typename: category,
+            pageval: 50,
+        },
+    });
+
+    const items = response.data.map((item) => ({
+        title: item.title,
+        link: `${rootUrl}/news/${item.id}`,
+        description: item.content,
+        pubDate: parseDate(item.time),
+        category: [item.keywords],
+        author: item.source,
+    }));
+
+    ctx.state.data = {
+        title: `深潮TechFlow - ${category}`,
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/techflow520/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/newsflash': ['nczitzk'],
+    '/:category?': ['nczitzk'],
+};

package/lib/v2/techflow520/newsflash.js

@@ -0,0 +1,32 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.techflow520.com';
+    const currentUrl = `${rootUrl}/newflash`;
+    const apiUrl = `${rootUrl}/index/kuaixun/load_kuaixun`;
+
+    const response = await got({
+        method: 'post',
+        url: apiUrl,
+        form: {
+            offect: 0,
+            pageval: 50,
+        },
+    });
+
+    const items = response.data.map((item) => ({
+        title: item.title,
+        link: `${rootUrl}/news/${item.id}`,
+        description: item.content,
+        pubDate: parseDate(item.time),
+        category: [item.keywords],
+        author: item.source,
+    }));
+
+    ctx.state.data = {
+        title: '深潮TechFlow - 快讯',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/techflow520/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'techflow520.com': {
+        _name: '深潮TechFlow',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/new-media.html#shen-chao-techflow-fen-lei',
+                source: ['/'],
+                target: '/techflow520/:category?',
+            },
+            {
+                title: '快讯',
+                docs: 'https://docs.rsshub.app/new-media.html#shen-chao-techflow-kuai-xun',
+                source: ['/newsflash'],
+                target: '/techflow520/newsflash',
+            },
+        ],
+    },
+};

package/lib/v2/techflow520/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/newsflash', require('./newsflash'));
+    router.get('/:category?', require('./index'));
+};

package/lib/v2/tencent/cloud/column.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? '86410';
+    const tag = ctx.params.tag ?? '';
+
+    const rootUrl = 'https://cloud.tencent.com';
+    const currentUrl = `${rootUrl}/developer/column/${id}${tag ? `/tag-${tag}` : ''}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const data = JSON.parse(response.data.match(/window\.\$render\((.*?)\);/)[1]);
+
+    let items = data.articleData.list.map((item) => ({
+        title: item.title,
+        author: item.author.name,
+        link: `${rootUrl}/developer/article/${item.id}`,
+        pubDate: parseDate(item.updateTime * 1000),
+        category: item.tags.map((tag) => tag.name),
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('.developer-code-block').remove();
+
+                item.description = content('.rno-markdown').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${data.columnDetail.name} - ${data.documentBaseTitle}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/tencent/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
     '/qq/sdk/changelog/:platform': ['nuomi1'],
+    '/cloud/column/:id?/:tag?': ['nczitzk'],
 };

package/lib/v2/tencent/radar.js

@@ -53,4 +53,15 @@ module.exports = {
             },
         ],
     },
+    'tencent.com': {
+        _name: '腾讯云',
+        '.': [
+            {
+                title: '云+社区专栏',
+                docs: 'https://docs.rsshub.app/programming.html#teng-xun-yun-yun-she-qu-zhuan-lan',
+                source: ['/developer/column/:id', '/developer/column/:id/:tag', '/'],
+                target: (params, url) => `/tencent/cloud/column/${url.match(/column\/(\d+)/)[1]}${/\/tag-\d+/.test(url) ? `/${url.match(/\/tag-(\d+)/)[1]}` : ''}`,
+            },
+        ],
+    },
 };

package/lib/v2/tencent/router.js

@@ -1,3 +1,4 @@
 module.exports = (router) => {
     router.get('/qq/sdk/changelog/:platform', require('./qq/sdk/changelog'));
+    router.get('/cloud/column/:id?/:tag?', require('./cloud/column'));
 };

package/lib/v2/test/index.js

@@ -98,6 +98,21 @@ module.exports = async (ctx) => {
             link: `https://mock.com/DIYgod/RSSHub`,
             author: `DIYgod`,
         });
+    } else if (ctx.params.id === 'multimedia') {
+        item.push({
+            title: `Multimedia Title`,
+            description: `<img src="/DIYgod/RSSHub.jpg">
+<video src="/DIYgod/RSSHub.mp4"></video>
+<video poster="/DIYgod/RSSHub.jpg">
+<source src="/DIYgod/RSSHub.mp4" type="video/mp4">
+<source src="/DIYgod/RSSHub.webm" type="video/webm">
+</video>
+<audio src="/DIYgod/RSSHub.mp3"></audio>
+<iframe src="/DIYgod/RSSHub.html"></iframe>`,
+            pubDate: new Date(`2019-3-1`).toUTCString(),
+            link: `https://mock.com/DIYgod/RSSHub`,
+            author: `DIYgod`,
+        });
     } else if (ctx.params.id === 'sort') {
         item.push({
             title: `Sort Title 0`,
@@ -187,6 +202,6 @@ module.exports = async (ctx) => {
         link: 'https://github.com/DIYgod/RSSHub',
         item,
         allowEmpty: ctx.params.id === 'allow_empty',
-        description: ctx.params.id === 'complicated' ? '<img src="http://mock.com/DIYgod/DIYgod/RSSHub">' : 'A test route for RSSHub',
+        description: ctx.params.id === 'complicated' ? '<img src="http://mock.com/DIYgod/DIYgod/RSSHub">' : ctx.params.id === 'multimedia' ? '<video src="http://mock.com/DIYgod/DIYgod/RSSHub"></video>' : 'A test route for RSSHub',
     };
 };

package/lib/v2/tiktok/user.js

@@ -1,5 +1,3 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
 const config = require('@/config').value;
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
@@ -10,57 +8,49 @@ const baseUrl = 'https://www.tiktok.com';
 module.exports = async (ctx) => {
     const { user } = ctx.params;
 
-    const html = await ctx.cache.tryGet(
+    const data = await ctx.cache.tryGet(
         `tiktok:user:${user}`,
         async () => {
             const browser = await require('@/utils/puppeteer')();
             const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
             await page.goto(`${baseUrl}/${user}`, {
-                waitUntil: 'networkidle2',
+                waitUntil: 'domcontentloaded',
             });
-            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            const SIGI_STATE = await page.evaluate(() => window.SIGI_STATE);
             browser.close();
-            return html;
+
+            const lang = SIGI_STATE.AppContext.lang;
+            const SharingMeta = SIGI_STATE.SharingMeta;
+            const ItemModule = SIGI_STATE.ItemModule;
+
+            return { lang, SharingMeta, ItemModule };
         },
         config.cache.routeExpire,
         false
     );
 
-    const $ = cheerio.load(html);
-
-    let items = $('[class*=-DivItemContainerV2]')
-        .toArray()
-        .map((item) => ({ link: $(item).find('[class*=-DivWrapper] a').attr('href') }));
-
-    items = await Promise.all(
-        items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const videoId = item.link.substring(item.link.lastIndexOf('/') + 1);
-                const api = (
-                    await got(`${baseUrl}/node/share/video/${user}/${videoId}`, {
-                        headers: {
-                            referer: `${baseUrl}/${user}`,
-                        },
-                    })
-                ).data;
-
-                item.title = api.itemInfo.itemStruct.desc;
-                item.description = art(path.join(__dirname, 'templates/user.art'), {
-                    poster: api.itemInfo.itemStruct.video.cover,
-                    source: api.itemInfo.itemStruct.video.playAddr,
-                });
-                item.author = api.itemInfo.itemStruct.author.nickname;
-                item.pubDate = parseDate(api.itemInfo.itemStruct.createTime * 1000);
-
-                return item;
-            })
-        )
-    );
+    const items = Object.values(data.ItemModule).map((item) => ({
+        title: item.desc,
+        description: art(path.join(__dirname, 'templates/user.art'), {
+            poster: item.video.cover,
+            source: item.video.playAddr,
+        }),
+        author: item.nickname,
+        pubDate: parseDate(item.createTime, 'X'),
+        link: `${baseUrl}/@${item.author}/video/${item.id}`,
+        category: item.textExtra.map((t) => `#${t.hashtagName}`),
+    }));
 
     ctx.state.data = {
-        title: $('title').text(),
-        description: $('[data-e2e=user-bio]').text(),
+        title: data.SharingMeta.value['og:title'],
+        description: data.SharingMeta.value['og:description'],
+        image: data.SharingMeta.value['og:image'],
         link: `${baseUrl}/${user}`,
         item: items,
+        language: data.lang,
     };
 };

package/lib/v2/trending/allTrending.js

@@ -0,0 +1,169 @@
+const dayjs = require('dayjs');
+dayjs.extend(require('dayjs/plugin/utc'));
+dayjs.extend(require('dayjs/plugin/timezone'));
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const config = require('@/config').value;
+const md5 = require('@/utils/md5');
+
+// Constants
+const DATA_REPO_BASE_URL = 'https://raw.githubusercontent.com/huqi-pr/trending-in-one/master/raw';
+const DATE_FORMAT = 'YYYY-MM-DD';
+// TODO: support custom data repo urls
+const CHANNELS = {
+    'toutiao-search': {
+        baseUrl: 'https://so.toutiao.com/search?keyword=',
+        name: '今日头条热搜',
+    },
+    'weibo-search': {
+        baseUrl: 'https://s.weibo.com/weibo?q=',
+        name: '微博热搜',
+    },
+    'zhihu-search': {
+        baseUrl: 'https://www.zhihu.com/search?q=',
+        name: '知乎热搜',
+    },
+    'zhihu-questions': {
+        baseUrl: 'https://www.zhihu.com/search?type=question&q=',
+        name: '知乎热门话题',
+    },
+    'zhihu-video': {
+        baseUrl: 'https://www.zhihu.com/search?type=video&q=',
+        name: '知乎热门视频',
+    },
+};
+
+// Helper Functions
+// TODO: integrate into CHANNELS
+const processRawDataByChannel = {
+    'toutiao-search': ({ word: title }) => ({
+        // 源 url 存在 encoding 问题，暂时不使用
+        url: CHANNELS['toutiao-search'].baseUrl + encodeURIComponent(title),
+        title,
+    }),
+    'weibo-search': ({ title }) => ({
+        // 源 url 存在 encoding 问题，暂时不使用
+        url: CHANNELS['weibo-search'].baseUrl + encodeURIComponent(title),
+        title,
+    }),
+    'zhihu-questions': (item) => item,
+    'zhihu-search': ({ query }) => {
+        const title = query.trim();
+        return {
+            // 源 url 存在 encoding 问题，暂时不使用
+            url: CHANNELS['zhihu-search'].baseUrl + encodeURIComponent(title),
+            title,
+        };
+    },
+    'zhihu-video': (item) => item,
+};
+
+const hasKeyword = (str, keywordList) => keywordList.some((keyword) => str.includes(keyword));
+const toShanghaiTimezone = (date) => dayjs.tz(date, 'Asia/Shanghai');
+
+// Data Fetcher
+// TODO: support channel selection
+const fetchAllData = async (keywordList = [], dateList = [], cache) => {
+    const cachedGetData = (url, dataMappingFn) =>
+        cache.tryGet(
+            url,
+            () =>
+                got(url)
+                    .json()
+                    // Normalize data to { title, url } format
+                    .then((res) => res.map((item) => dataMappingFn(item))),
+            config.cache.contentExpire,
+            false
+        );
+
+    const data = await Promise.all(
+        dateList.map(async (dateTime) => ({
+            dateTime,
+            data: await Promise.all(
+                Object.keys(CHANNELS).map(async (channel) => ({
+                    name: CHANNELS[channel].name,
+                    data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`, processRawDataByChannel[channel]).then((res) => res.filter(({ title }) => hasKeyword(title, keywordList))),
+                }))
+            ),
+        }))
+    );
+    for (const i of data) {
+        i.count = i.data.reduce((acc, { data }) => acc + data.length, 0);
+    }
+
+    return data.filter(({ count }) => count > 0);
+};
+
+// Generate Feed Items
+const searchLinkUrls = (keyword) => [
+    `https://tophub.today/search?e=tophub&q=${keyword}`,
+    `https://www.baidu.com/s?wd=${keyword}`,
+    `https://www.google.com/search?q=${keyword}`,
+    `https://www.zhihu.com/search?type=content&q=${keyword}`,
+    `https://s.weibo.com/weibo/${keyword}`,
+    `https://www.douyin.com/search/${keyword}`,
+    `https://so.toutiao.com/search?keyword=${keyword}`,
+];
+
+const searchLinkNames = ['热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条'];
+
+const createItem = ({ dateTime, data, count }, keywords, isToday) => {
+    const EOD = dateTime.endOf('day');
+    const pubDate = isToday ? new Date() : EOD.toDate();
+    return {
+        title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${count})`,
+        author: 'Trending All In One',
+        pubDate,
+        description: art(path.join(__dirname, 'templates/content.art'), {
+            data,
+            queries: keywords.map((query) => ({
+                links: searchLinkUrls(encodeURIComponent(query)).map((url, index) => `<a href="${url}" rel="noopener noreferrer" target="_blank">${searchLinkNames[index]}</a>`),
+                key: query,
+            })),
+        }),
+        guid: `trending-all-in-one-${EOD.toISOString()}-${md5(JSON.stringify(data))}-${keywords.join('-')}`,
+    };
+};
+
+// Main
+module.exports = async (ctx) => {
+    // Prevent making over 100 requests per invocation
+    if (ctx.params.numberOfDays > 14) {
+        throw new Error('days must be less than 14');
+    }
+    const numberOfDays = ctx.params.numberOfDays || 3;
+    const currentShanghaiDateTime = dayjs(toShanghaiTimezone(new Date()));
+    const currentShanghaiDateStr = currentShanghaiDateTime.format(DATE_FORMAT);
+    const dateList = [];
+    for (let i = 0; i < numberOfDays; i++) {
+        const d = currentShanghaiDateTime.subtract(i, 'day');
+        dateList.push(d);
+    }
+
+    const keywordList = ctx.params.keywords
+        .replace('，', ',')
+        .split(',')
+        .map((keyword) => keyword.trim());
+    const keywordStr = keywordList.join(', ');
+
+    const data = await fetchAllData(keywordList, dateList, ctx.cache);
+    const item =
+        data.length > 0
+            ? data.map((i, index) => createItem(i, keywordList, index === 0))
+            : [
+                  {
+                      title: `${keywordStr} | ${currentShanghaiDateStr} 热点追踪 (0)`,
+                      author: 'Trending All In One',
+                      description: `近${numberOfDays}日的热搜都不包含下列关键词：${keywordStr}<br>请耐心等待，或添加更多关键词试试。`,
+                      guid: `trending-all-in-one-${md5(JSON.stringify(data))}-${keywordList.join('-')}`,
+                  },
+              ];
+
+    ctx.state.data = {
+        title: `${keywordStr} | 热点聚合`,
+        description: `${keywordStr} | 今日头条热搜，知乎热门视频，知乎热搜榜，知乎热门话题，微博热搜榜聚合追踪`,
+        language: 'zh-cn',
+        item,
+    };
+};

package/lib/v2/trending/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:keywords/:numberOfDays?': ['Jkker'],
+};