rsshub 1.0.0-master.f97cdb2 → 1.0.0-master.fa3428f

package/lib/v2/thecover/channel.js

@@ -0,0 +1,66 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const rootUrl = 'https://www.thecover.cn';
+
+const nodes = {
+    3892: '天下',
+    3560: '四川',
+    3909: '辟谣',
+    3686: '国际',
+    11: '云招考',
+    3902: '30秒',
+    3889: '拍客',
+    3689: '体育',
+    1: '国内',
+    4002: '帮扶铁军',
+    12: '文娱',
+    46: '宽窄',
+    4: '商业',
+    21: '千面',
+    17: '封面号',
+};
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? '3892';
+    const targetUrl = rootUrl.concat(`/channel_${id}`);
+    const resp = await got({
+        method: 'get',
+        url: targetUrl,
+    });
+    const $ = cheerio.load(resp.data);
+    const list = $('a.link-to-article')
+        .filter(function () {
+            return $(this).attr('href').startsWith('/');
+        })
+        .map((_, item) => ({
+            link: rootUrl.concat($(item).attr('href')),
+        }))
+        .get();
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(detailResponse.data);
+                item.title = content('h1', '.main-article').text();
+                item.description = content('section.article-content').html();
+                const info = content('span', '.props-of-title');
+                item.author = info.eq(0).text();
+                item.pubDate = timezone(parseDate(info.eq(1).text(), 'YYYY-MM-DD HH:mm'), +8);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${nodes[id]}-封面新闻`,
+        link: targetUrl,
+        description: `封面新闻作为华西都市报深度融合转型和打造新型主流媒体的载体，牢固确立移动优先战略，创新移动新闻产品，打造移动传播矩阵，封面新闻的传播力、引导力、影响力和公信力不断得到各方肯定。封面新闻突破千万的用户下载量，呈现出以四川为主阵地的全国分布态势，用户年龄构成以20-35岁为主，“亿万年轻人的生活方式”的定位初步得到体现。`,
+        language: 'zh-cn',
+        item: items,
+    };
+};

package/lib/v2/thecover/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/channel/:id?': ['yuxinliu-alex'],
+};

package/lib/v2/thecover/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'thecover.cn': {
+        _name: '封面新闻',
+        '.': [
+            {
+                title: '频道',
+                docs: 'https://docs.rsshub.app/new-media.html#the-cover',
+                source: ['/:id', '/'],
+                target: (params) => `/thecover/channel/${params.id.replace('channel_', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/thecover/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/channel/:id?', require('./channel'));
+};

package/lib/{routes → v2}/twreporter/category.js

@@ -6,26 +6,21 @@ const fetch = require('./fetch_article');
 module.exports = async (ctx) => {
     const baseURL = 'https://www.twreporter.org';
     const url = baseURL + `/categories/${ctx.params.cid}`;
-    const res = await got.get(url);
+    const res = await got(url);
     const $ = cheerio.load(res.data);
     const list = $('.lnKPLr').get();
     const category = $('.kCfkTU').text();
 
     const out = await Promise.all(
-        list.map(async (item) => {
+        list.map((item) => {
             const $ = cheerio.load(item);
             const address = baseURL + $('a').attr('href');
             const title = $('.list-item__Title-sc-1dx5lew-5').text();
-            const cache = await ctx.cache.get(address);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const single = await fetch(address);
-            single.title = title;
-
-            ctx.cache.set(address, JSON.stringify(single));
-            return Promise.resolve(single);
+            return ctx.cache.tryGet(address, async () => {
+                const single = await fetch(address);
+                single.title = title;
+                return single;
+            });
         })
     );
     ctx.state.data = {

package/lib/{routes → v2}/twreporter/fetch_article.js

@@ -1,18 +1,19 @@
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async function fetch(address) {
-    const res = await got.get(address);
+    const res = await got(address);
     const capture = cheerio.load(res.data);
     capture('.gIMvvS').remove();
 
-    let metaInfoBox = capture('.iRdvCH')
+    let metaInfoBox = capture('.ffAPnj')
         .filter((index) => index === 0)
         .get();
 
     // For photography
     if (metaInfoBox.length === 0) {
-        metaInfoBox = capture('.dPQBaW')
+        metaInfoBox = capture('.deNvJY')
             .filter((index) => index === 0)
             .get();
     }
@@ -61,6 +62,6 @@ module.exports = async function fetch(address) {
         description: contents,
         link: address,
         guid: address,
-        pubDate: new Date(time).toUTCString(),
+        pubDate: parseDate(time, 'M/D/YYYY'),
     };
 };

package/lib/v2/twreporter/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/category/:cid': ['emdoe'],
+    '/newest': ['emdoe'],
+    '/photography': ['emdoe'],
+};

package/lib/{routes → v2}/twreporter/newest.js

@@ -5,25 +5,20 @@ const fetch = require('./fetch_article');
 
 module.exports = async (ctx) => {
     const url = 'https://www.twreporter.org';
-    const res = await got.get(url);
+    const res = await got(url);
     const $ = cheerio.load(res.data);
     const list = $('.gKMjSz').get();
 
     const out = await Promise.all(
-        list.map(async (item) => {
+        list.map((item) => {
             const $ = cheerio.load(item);
             const address = url + $('a').attr('href');
-            const title = $('.dpNivU').text();
-            const cache = await ctx.cache.get(address);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const single = await fetch(address);
-            single.title = title;
-
-            ctx.cache.set(address, JSON.stringify(single));
-            return Promise.resolve(single);
+            const title = $('.latest-section__Title-hzxpx3-6').text();
+            return ctx.cache.tryGet(address, async () => {
+                const single = await fetch(address);
+                single.title = title;
+                return single;
+            });
         })
     );
     ctx.state.data = {

package/lib/v2/twreporter/photography.js

@@ -0,0 +1,46 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+const fetch = require('./fetch_article');
+
+module.exports = async (ctx) => {
+    const baseURL = 'https://www.twreporter.org';
+    const url = baseURL + `/photography`;
+    const res = await got(url);
+    const $ = cheerio.load(res.data);
+    const coverList = $('.WPJvn').get();
+    const commonList = $('.eVNsZf').get();
+
+    const coverView = await Promise.all(
+        coverList.map((item) => {
+            const $ = cheerio.load(item);
+            const address = baseURL + $('li > a').attr('href');
+            const title = $('.sc-1aojo4z-4').text();
+            return ctx.cache.tryGet(address, async () => {
+                const single = await fetch(address);
+                single.title = title;
+                return single;
+            });
+        })
+    );
+
+    const listView = await Promise.all(
+        commonList.map((item) => {
+            const $ = cheerio.load(item);
+            const address = baseURL + $('li > a').attr('href');
+            const title = $('.ii0887-4').text();
+
+            return ctx.cache.tryGet(address, async () => {
+                const single = await fetch(address);
+                single.title = title;
+                return single;
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: `報導者 | 影像`,
+        link: url,
+        item: coverView.concat(listView),
+    };
+};

package/lib/v2/twreporter/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'twreporter.org': {
+        _name: '報導者',
+        '.': [
+            {
+                title: '最新',
+                docs: 'https://docs.rsshub.app/new-media.html#bao-dao-zhe',
+                source: ['/'],
+                target: '/twreporter/newest',
+            },
+            {
+                title: '摄影',
+                docs: 'https://docs.rsshub.app/new-media.html#bao-dao-zhe',
+                source: ['/photography'],
+                target: '/twreporter',
+            },
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/new-media.html#bao-dao-zhe',
+                source: ['/categories/:tid'],
+                target: '/twreporter/category/:tid',
+            },
+        ],
+    },
+};

package/lib/v2/twreporter/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/category/:cid', require('./category'));
+    router.get('/newest', require('./newest'));
+    router.get('/photography', require('./photography'));
+};

package/lib/{routes/tencent → v2}/wechat/announce.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const { data: htmlString } = await got({
@@ -20,13 +21,13 @@ module.exports = async (ctx) => {
             title: `${time} ${title}`,
             link: `https://mp.weixin.qq.com${$link.attr('href')}`,
             description: title,
-            pubDate: new Date(time).toUTCString(),
+            pubDate: parseDate(time),
         });
     });
 
     ctx.state.data = {
         title: '微信公众平台-系统公告栏目',
-        link: 'https://mp.weixin.qq.com/cgi-bin/announce?action=getannouncementlist&lang=zh_CN&token=',
+        link: 'https://mp.weixin.qq.com/cgi-bin/announce?action=getannouncementlist&lang=zh_CN',
         item: announceList,
     };
 };

package/lib/v2/wechat/ce.js

@@ -0,0 +1,69 @@
+const parser = require('@/utils/rss-parser');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { fixArticleContent } = require('@/utils/wechat-mp');
+const { parseDate } = require('@/utils/parse-date');
+
+// any UA containing "RSS" can pass the check
+// mark the UA as a desktop UA with "(X11; Linux x86_64)"
+const UA = 'Mozilla/5.0 (X11; Linux x86_64) RSS Reader';
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+
+    const feed = await parser.parseString(
+        await got
+            .get(`https://posts.careerengine.us/author/${id}/rss`, {
+                headers: {
+                    'User-Agent': UA,
+                },
+            })
+            .then((_) => _.data)
+    );
+
+    const items = await Promise.all(
+        feed.items.splice(0, 10).map(async (item) => {
+            // generally speaking, changing `item.link` of an existing route could potentially break `item.guid`
+            // but since the route has been down for at least 8 months, it's probably safe
+            item.link = item.link.replace(/^http:\/\//, 'https://');
+            return await ctx.cache.tryGet(item.link, async () => {
+                const response = await got.get(item.link, {
+                    headers: {
+                        'User-Agent': UA,
+                    },
+                });
+
+                const $ = cheerio.load(response.data);
+
+                const description = fixArticleContent($('.post'));
+
+                let pubDate = item.pubDate;
+                if (!pubDate || pubDate === 'Invalid Date') {
+                    // sometimes the pubDate is not available in the official feed
+                    const postDate = $('.post-date')
+                        .text()
+                        .replace(/\s+|发表/g, '');
+                    // the date format is "发表 YYYY年MM月DD日 "
+                    // following the official feed behavior: imprecise date is in UTC
+                    // `<pubDate>Mon, 04 Apr 2022 00:00:00 GMT</pubDate>`
+                    pubDate = parseDate(postDate, 'YYYY年MM月DD日');
+                    pubDate = new Date(pubDate.getTime() - pubDate.getTimezoneOffset() * 60 * 1000);
+                }
+
+                return {
+                    title: item.title,
+                    description,
+                    pubDate,
+                    link: item.link,
+                };
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: `微信公众号 - ${feed.title}`,
+        link: `https://posts.careerengine.us/author/${id}/posts`,
+        description: feed.description,
+        item: items,
+    };
+};

package/lib/v2/wechat/data258.js

@@ -0,0 +1,137 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+const { RequestInProgressError } = require('@/errors');
+const wait = require('@/utils/wait');
+
+const parsePage = ($item, hyperlinkSelector, timeSelector) => {
+    const hyperlink = $item.find(hyperlinkSelector);
+    const title = hyperlink.text();
+    const link = hyperlink.attr('href');
+    const pubDate = timezone(parseDate($item.find(timeSelector).text(), 'YYYY-MM-DD HH:mm'), 8);
+    return {
+        title,
+        link,
+        pubDate,
+    };
+};
+
+module.exports = async (ctx) => {
+    // !!! here we must use a lock to prevent other requests to break the anti-anti-crawler workarounds !!!
+    if ((await ctx.cache.get('data258:lock', false)) === '1') {
+        throw new RequestInProgressError('Another request is in progress, please try again later.');
+    }
+    // !!! here no need to acquire the lock, because the MP/category page has no crawler detection !!!
+
+    const id = ctx.params.id;
+
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 5;
+
+    const rootUrl = 'https://mp.data258.com';
+    const pageUrl = id ? `${rootUrl}/article/category/${id}` : rootUrl;
+
+    const response = await got(pageUrl);
+    const $ = cheerio.load(response.data);
+
+    const title = $('head title').text();
+    // title = title.endsWith('-微阅读') ? title.slice(0, title.length - 4) : title;
+    const description = $('meta[name="description"]').attr('content');
+
+    const categoryPage = $('ul.fly-list');
+
+    let items;
+    if (categoryPage && categoryPage.length) {
+        // got a category page
+        items = $(categoryPage)
+            .find('li')
+            .map((_, item) => parsePage($(item), 'h2 a', '.fly-list-info span'))
+            .get();
+    } else {
+        // got an MP page
+        items = $('ul.jie-row li')
+            .map((_, item) => parsePage($(item), 'a.jie-title', '.layui-hide-xs'))
+            .get();
+    }
+
+    items = items.slice(0, limit); // limit to avoid being anti-crawled
+
+    // !!! double-check !!!
+    if ((await ctx.cache.get('data258:lock', false)) === '1') {
+        throw new RequestInProgressError('Another request is in progress, please try again later.');
+    } else {
+        // !!! here we acquire the lock because the jump page has crawler detection !!!
+        await ctx.cache.set('data258:lock', '1', 60);
+    }
+
+    // !!! here we must use a for-loop to ensure the concurrency is 1 !!!
+    // !!! please do note that if you try to increase the concurrency, your IP will be banned for a long time !!!
+
+    let err; // !!! let RSSHub throw an anti-crawler prompt if the route is empty !!!
+
+    /* eslint-disable no-await-in-loop */
+    for (const item of items) {
+        // https://mp.data258.com/wx?id=${id}&t={token}, id is a permanent hex, token is a temporary base64
+        const cacheId = item.link.match(/id=([\da-f]+)/)[1];
+        item.link = item.link.startsWith('http') ? item.link : `${rootUrl}${item.link}`;
+        const realLink = await ctx.cache.tryGet(`data258:${cacheId}`, async () => {
+            try {
+                // !!! here we must sleep 1s to avoid being anti-crawled !!!
+                // !!! please do note that if the interval is less than 1s, your IP will be banned for a long time !!!
+                await wait(1000);
+
+                const response = await got.get(item.link, {
+                    headers: {
+                        Referer: pageUrl, // essential
+                    },
+                });
+                if (response.data.includes('今日浏览次数已达上限')) {
+                    // !!! as long as cache hits, the link will not be crawled and consume the limit !!!
+                    // !!! so that's not a big problem if the RSSHub instance is self-hosted !!!
+                    err = new got.RequestError(response.data, {}, response.request);
+                    return null;
+                }
+                const $ = cheerio.load(response.data);
+                const jmpJS = $('script')
+                    .filter((_, e) => $(e).html().includes('location.href'))
+                    .html();
+                return jmpJS.match(/location\.href='([^']+)'/)[1];
+            } catch (e) {
+                err = e;
+                return null;
+            }
+        });
+        if (realLink) {
+            item.link = realLink;
+        } else {
+            break; // being anti-crawled, immediately cancel following operations
+        }
+    }
+    /* eslint-enable no-await-in-loop */
+
+    // !!! release the lock, let it expire immediately since no need to keep it in cache !!!
+    await ctx.cache.set('data258:lock', '0', 1);
+
+    // jump links are valid only for a short period of time, drop those un-jumped items
+    // http://mp.weixin.qq.com/s
+    items = items.filter((item) => item.link.match(/^https?:\/\/mp\.weixin\.qq\.com\/s/));
+
+    if (items.length === 0 && err) {
+        // !!! if each request is anti-crawled, the filtered items array will be empty !!!
+        // !!! let RSSHub throw an anti-crawler prompt !!!
+        throw err;
+    }
+
+    await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item, !!categoryPage)));
+
+    ctx.state.data = {
+        title,
+        link: pageUrl,
+        description,
+        item: items,
+    };
+};
+
+// TODO: login? the valid time for cookies seems to be short, and abusing account will probably get banned...
+// TODO: fetch full article for the official RSS feed? unless someone who is VIP contributes their RSS feed for test...

package/lib/v2/wechat/ershcimi.js

@@ -0,0 +1,35 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const rootUrl = 'https://www.cimidata.com';
+
+    const url = `${rootUrl}/a/${id}`;
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const items = $('.weui_media_box')
+        .map((_, ele) => {
+            const $item = cheerio.load(ele);
+            const link = $item('.weui_media_title a').attr('href');
+            return {
+                title: $item('.weui_media_title a').text(),
+                description: $item('.weui_media_desc').text(),
+                link,
+                pubDate: timezone(parseDate($item('.weui_media_extra_info').attr('title')), +8),
+            };
+        })
+        .get();
+
+    await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item)));
+
+    ctx.state.data = {
+        title: `微信公众号 - ${$('span.name').text()}`,
+        link: url,
+        description: $('div.Profile-sideColumnItemValue').text(),
+        item: items,
+    };
+};

package/lib/v2/wechat/maintainer.js

@@ -1,3 +1,14 @@
 module.exports = {
+    '/announce': ['xyqfer'],
+    '/ce/:id': ['HenryQW'],
+    '/ershicimi/:id': ['sanmmm'],
+    '/data258/:id?': ['Rongronggg9'],
     '/feeddd/:id': ['TonyRL', 'Rongronggg9'],
+    '/feeds': ['tylinux'],
+    '/mp/homepage/:biz/:hid/:cid?': ['MisteryMonster'],
+    '/mp/msgalbum/:biz/:aid': ['MisteryMonster'],
+    '/tgchannel/:id/:mpName?/:searchQueryType?': ['LogicJake', 'Rongronggg9'],
+    '/uread/:userid': ['kt286'],
+    '/wemp/:id': ['HenryQW'],
+    '/wxnmh/:id': ['laampui'],
 };

package/lib/v2/wechat/radar.js

@@ -1,12 +1,87 @@
 module.exports = {
+    'careerengine.us': {
+        _name: '微信',
+        posts: [
+            {
+                title: '公众号（CareerEngine 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/author/*id/posts'],
+                target: (params) => `/wechat/ce/${params.id}`,
+            },
+        ],
+    },
+    'cimidata.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（二十次幂来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/a/:id'],
+                target: '/wechat/ce/:id',
+            },
+        ],
+    },
+    'data258.com': {
+        _name: '微信',
+        mp: [
+            {
+                title: '公众号（微阅读来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/', '/article/category/:id'],
+                target: '/wechat/data258/:id?',
+            },
+        ],
+    },
     'feeddd.org': {
         _name: '微信',
         '.': [
             {
-                title: '公众号 (feeddd 来源)',
+                title: '公众号（feeddd 来源）',
                 docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
                 source: ['/'],
             },
         ],
     },
+    'mp.weixin.qq.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众平台系统公告栏目',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/cgi-bin/announce'],
+                target: '/wechat/announce',
+            },
+        ],
+    },
+    'privacyhide.com': {
+        _name: '微信',
+        wechat: [
+            {
+                title: '公众号（wechat-feeds 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+            },
+        ],
+    },
+    'wemp.app': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（wemp.app 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/accounts/:id'],
+                target: '/wechat/wemp/:id',
+            },
+        ],
+    },
+    'wxnmh.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（wxnmh.com 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/:id'],
+                target: (params) => `/wechat/wxnmh/${params.id.replace('user-', '').replace('.htm', '')}`,
+            },
+        ],
+    },
 };