rsshub 1.0.0-master.ff1cc11 → 1.0.0-master.ff97759

package/lib/v2/wechat/ce.js

@@ -0,0 +1,69 @@
+const parser = require('@/utils/rss-parser');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { fixArticleContent } = require('@/utils/wechat-mp');
+const { parseDate } = require('@/utils/parse-date');
+
+// any UA containing "RSS" can pass the check
+// mark the UA as a desktop UA with "(X11; Linux x86_64)"
+const UA = 'Mozilla/5.0 (X11; Linux x86_64) RSS Reader';
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+
+    const feed = await parser.parseString(
+        await got
+            .get(`https://posts.careerengine.us/author/${id}/rss`, {
+                headers: {
+                    'User-Agent': UA,
+                },
+            })
+            .then((_) => _.data)
+    );
+
+    const items = await Promise.all(
+        feed.items.splice(0, 10).map(async (item) => {
+            // generally speaking, changing `item.link` of an existing route could potentially break `item.guid`
+            // but since the route has been down for at least 8 months, it's probably safe
+            item.link = item.link.replace(/^http:\/\//, 'https://');
+            return await ctx.cache.tryGet(item.link, async () => {
+                const response = await got.get(item.link, {
+                    headers: {
+                        'User-Agent': UA,
+                    },
+                });
+
+                const $ = cheerio.load(response.data);
+
+                const description = fixArticleContent($('.post'));
+
+                let pubDate = item.pubDate;
+                if (!pubDate || pubDate === 'Invalid Date') {
+                    // sometimes the pubDate is not available in the official feed
+                    const postDate = $('.post-date')
+                        .text()
+                        .replace(/\s+|发表/g, '');
+                    // the date format is "发表 YYYY年MM月DD日 "
+                    // following the official feed behavior: imprecise date is in UTC
+                    // `<pubDate>Mon, 04 Apr 2022 00:00:00 GMT</pubDate>`
+                    pubDate = parseDate(postDate, 'YYYY年MM月DD日');
+                    pubDate = new Date(pubDate.getTime() - pubDate.getTimezoneOffset() * 60 * 1000);
+                }
+
+                return {
+                    title: item.title,
+                    description,
+                    pubDate,
+                    link: item.link,
+                };
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: `微信公众号 - ${feed.title}`,
+        link: `https://posts.careerengine.us/author/${id}/posts`,
+        description: feed.description,
+        item: items,
+    };
+};

package/lib/v2/wechat/data258.js

@@ -0,0 +1,137 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+const { RequestInProgressError } = require('@/errors');
+const wait = require('@/utils/wait');
+
+const parsePage = ($item, hyperlinkSelector, timeSelector) => {
+    const hyperlink = $item.find(hyperlinkSelector);
+    const title = hyperlink.text();
+    const link = hyperlink.attr('href');
+    const pubDate = timezone(parseDate($item.find(timeSelector).text(), 'YYYY-MM-DD HH:mm'), 8);
+    return {
+        title,
+        link,
+        pubDate,
+    };
+};
+
+module.exports = async (ctx) => {
+    // !!! here we must use a lock to prevent other requests to break the anti-anti-crawler workarounds !!!
+    if ((await ctx.cache.get('data258:lock', false)) === '1') {
+        throw new RequestInProgressError('Another request is in progress, please try again later.');
+    }
+    // !!! here no need to acquire the lock, because the MP/category page has no crawler detection !!!
+
+    const id = ctx.params.id;
+
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 5;
+
+    const rootUrl = 'https://mp.data258.com';
+    const pageUrl = id ? `${rootUrl}/article/category/${id}` : rootUrl;
+
+    const response = await got(pageUrl);
+    const $ = cheerio.load(response.data);
+
+    const title = $('head title').text();
+    // title = title.endsWith('-微阅读') ? title.slice(0, title.length - 4) : title;
+    const description = $('meta[name="description"]').attr('content');
+
+    const categoryPage = $('ul.fly-list');
+
+    let items;
+    if (categoryPage && categoryPage.length) {
+        // got a category page
+        items = $(categoryPage)
+            .find('li')
+            .map((_, item) => parsePage($(item), 'h2 a', '.fly-list-info span'))
+            .get();
+    } else {
+        // got an MP page
+        items = $('ul.jie-row li')
+            .map((_, item) => parsePage($(item), 'a.jie-title', '.layui-hide-xs'))
+            .get();
+    }
+
+    items = items.slice(0, limit); // limit to avoid being anti-crawled
+
+    // !!! double-check !!!
+    if ((await ctx.cache.get('data258:lock', false)) === '1') {
+        throw new RequestInProgressError('Another request is in progress, please try again later.');
+    } else {
+        // !!! here we acquire the lock because the jump page has crawler detection !!!
+        await ctx.cache.set('data258:lock', '1', 60);
+    }
+
+    // !!! here we must use a for-loop to ensure the concurrency is 1 !!!
+    // !!! please do note that if you try to increase the concurrency, your IP will be banned for a long time !!!
+
+    let err; // !!! let RSSHub throw an anti-crawler prompt if the route is empty !!!
+
+    /* eslint-disable no-await-in-loop */
+    for (const item of items) {
+        // https://mp.data258.com/wx?id=${id}&t={token}, id is a permanent hex, token is a temporary base64
+        const cacheId = item.link.match(/id=([\da-f]+)/)[1];
+        item.link = item.link.startsWith('http') ? item.link : `${rootUrl}${item.link}`;
+        const realLink = await ctx.cache.tryGet(`data258:${cacheId}`, async () => {
+            try {
+                // !!! here we must sleep 1s to avoid being anti-crawled !!!
+                // !!! please do note that if the interval is less than 1s, your IP will be banned for a long time !!!
+                await wait(1000);
+
+                const response = await got.get(item.link, {
+                    headers: {
+                        Referer: pageUrl, // essential
+                    },
+                });
+                if (response.data.includes('今日浏览次数已达上限')) {
+                    // !!! as long as cache hits, the link will not be crawled and consume the limit !!!
+                    // !!! so that's not a big problem if the RSSHub instance is self-hosted !!!
+                    err = new got.RequestError(response.data, {}, response.request);
+                    return null;
+                }
+                const $ = cheerio.load(response.data);
+                const jmpJS = $('script')
+                    .filter((_, e) => $(e).html().includes('location.href'))
+                    .html();
+                return jmpJS.match(/location\.href='([^']+)'/)[1];
+            } catch (e) {
+                err = e;
+                return null;
+            }
+        });
+        if (realLink) {
+            item.link = realLink;
+        } else {
+            break; // being anti-crawled, immediately cancel following operations
+        }
+    }
+    /* eslint-enable no-await-in-loop */
+
+    // !!! release the lock, let it expire immediately since no need to keep it in cache !!!
+    await ctx.cache.set('data258:lock', '0', 1);
+
+    // jump links are valid only for a short period of time, drop those un-jumped items
+    // http://mp.weixin.qq.com/s
+    items = items.filter((item) => item.link.match(/^https?:\/\/mp\.weixin\.qq\.com\/s/));
+
+    if (items.length === 0 && err) {
+        // !!! if each request is anti-crawled, the filtered items array will be empty !!!
+        // !!! let RSSHub throw an anti-crawler prompt !!!
+        throw err;
+    }
+
+    await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item, !!categoryPage)));
+
+    ctx.state.data = {
+        title,
+        link: pageUrl,
+        description,
+        item: items,
+    };
+};
+
+// TODO: login? the valid time for cookies seems to be short, and abusing account will probably get banned...
+// TODO: fetch full article for the official RSS feed? unless someone who is VIP contributes their RSS feed for test...

package/lib/v2/wechat/ershcimi.js

@@ -0,0 +1,35 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const rootUrl = 'https://www.cimidata.com';
+
+    const url = `${rootUrl}/a/${id}`;
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const items = $('.weui_media_box')
+        .map((_, ele) => {
+            const $item = cheerio.load(ele);
+            const link = $item('.weui_media_title a').attr('href');
+            return {
+                title: $item('.weui_media_title a').text(),
+                description: $item('.weui_media_desc').text(),
+                link,
+                pubDate: timezone(parseDate($item('.weui_media_extra_info').attr('title')), +8),
+            };
+        })
+        .get();
+
+    await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item)));
+
+    ctx.state.data = {
+        title: `微信公众号 - ${$('span.name').text()}`,
+        link: url,
+        description: $('div.Profile-sideColumnItemValue').text(),
+        item: items,
+    };
+};

package/lib/v2/wechat/feeddd.js

@@ -1,8 +1,6 @@
 const got = require('@/utils/got');
-const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-const { art } = require('@/utils/render');
-const path = require('path');
+const { finishArticleItem } = require('@/utils/wechat-mp');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
@@ -12,55 +10,22 @@ module.exports = async (ctx) => {
 
     const response = await got(apiUrl);
 
-    if (response.data.items.length === 0) {
-        throw Error('Empty feed');
-    }
-
     let items = response.data.items.map((item) => ({
         title: item.title,
+        // the date is when the article was grabbed, not published, `finishArticleItem` will fix this
         pubDate: parseDate(item.date_modified),
         link: item.url,
-        author: item.title,
         guid: item.id,
     }));
 
-    items = await Promise.all(
-        items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    url: item.link,
-                    headers: {
-                        Referer: 'https://mp.weixin.qq.com',
-                    },
-                });
-
-                const $ = cheerio.load(detailResponse.data);
-
-                // fix lazyload image
-                $('img').each((_, e) => {
-                    e = $(e);
-                    e.after(
-                        art(path.join(__dirname, 'templates/image.art'), {
-                            src: e.attr('data-src') ?? e.attr('src'),
-                        })
-                    );
-                    e.remove();
-                });
-
-                item.description = art(path.join(__dirname, 'templates/description.art'), {
-                    desc: $('#js_content').html(),
-                });
-
-                return item;
-            })
-        )
-    );
+    items = await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item)));
 
     ctx.state.data = {
         title: response.data.title,
         link: response.data.feed_url,
         description: response.data.title,
         item: items,
+        allowEmpty: true,
     };
 
     ctx.state.json = {

package/lib/v2/wechat/feeds.js

@@ -0,0 +1,23 @@
+const parser = require('@/utils/rss-parser');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const link = `https://github.com/hellodword/wechat-feeds/raw/feeds/${id}.xml`;
+    const feed = await parser.parseURL(link);
+
+    const items = feed.items.map((item) => ({
+        title: item.title,
+        pubDate: new Date(item.pubDate),
+        link: item.link,
+        guid: item.link,
+    }));
+    await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item)));
+
+    ctx.state.data = {
+        title: feed.title,
+        link,
+        description: feed.description,
+        item: items,
+    };
+};

package/lib/v2/wechat/maintainer.js

@@ -1,3 +1,14 @@
 module.exports = {
-    '/feeddd/:id': ['TonyRL'],
+    '/announce': ['xyqfer'],
+    '/ce/:id': ['HenryQW'],
+    '/ershicimi/:id': ['sanmmm'],
+    '/data258/:id?': ['Rongronggg9'],
+    '/feeddd/:id': ['TonyRL', 'Rongronggg9'],
+    '/feeds': ['tylinux'],
+    '/mp/homepage/:biz/:hid/:cid?': ['MisteryMonster'],
+    '/mp/msgalbum/:biz/:aid': ['MisteryMonster'],
+    '/tgchannel/:id/:mpName?/:searchQueryType?': ['LogicJake', 'Rongronggg9'],
+    '/uread/:userid': ['kt286'],
+    '/wemp/:id': ['HenryQW'],
+    '/wxnmh/:id': ['laampui'],
 };

package/lib/v2/wechat/mp.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const dayjs = require('dayjs');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+
+module.exports = async (ctx) => {
+    const { biz, hid, cid } = ctx.params;
+    let cidurl = '';
+    if (cid) {
+        cidurl = `&cid=${cid}`;
+    }
+    let hidurl = '';
+    if (hid) {
+        hidurl = `&hid=${hid}`;
+    }
+    const JSONresponse = await got({
+        method: 'post',
+        url: `https://mp.weixin.qq.com/mp/homepage?__biz=${biz}${hidurl}${cidurl}&begin=0&count=5&action=appmsg_list`,
+    });
+    // 主页Html，获取 RSS 标题用
+    const HTMLresponse = await got({
+        method: 'get',
+        url: `https://mp.weixin.qq.com/mp/homepage?__biz=${biz}${hidurl}${cidurl}`,
+    });
+    const list = JSONresponse.data.appmsg_list;
+    const $ = cheerio.load(HTMLresponse.data);
+    // 标题，另外差一个菜单标题！求助
+    const mptitle = $('div.articles_header').find('a').text() + `|` + $('div.articles_header > h2.rich_media_title').text();
+    const articledata = await Promise.all(
+        list.map(async (item) => {
+            const single = {
+                link: item.link,
+                guid: item.link,
+            };
+            return await finishArticleItem(ctx, single);
+        })
+    );
+    ctx.state.data = {
+        // 源标题，差一个cid相关的标题！
+        title: mptitle,
+        link: `https://mp.weixin.qq.com/mp/homepage?__biz=${biz}${hidurl}${cidurl}`,
+        item: list.map((item, index) => ({
+            title: item.title,
+            description: `
+                ${item.digest}<br>
+                <img
+                    style="max-width: 650px; height: auto; object-fit: contain; width: 100%;"
+                    src="${item.cover}"
+                ><br>
+                <br>
+                ${articledata[index].description}
+            `,
+            link: articledata[index].link,
+            guid: articledata[index].guid,
+            author: articledata[index].author,
+            pubDate: dayjs.unix(item.sendtime).format(),
+        })),
+    };
+};

package/lib/v2/wechat/msgalbum.js

@@ -0,0 +1,41 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const dayjs = require('dayjs');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+
+module.exports = async (ctx) => {
+    const { biz, aid } = ctx.params;
+    const aidurl = `&album_id=${aid}`;
+
+    const HTMLresponse = await got({
+        method: 'get',
+        url: `https://mp.weixin.qq.com/mp/appmsgalbum?__biz=${biz}&action=getalbum${aidurl}`,
+    });
+    const $ = cheerio.load(HTMLresponse.data);
+    const list = $('li').get();
+    const mptitle = $('.album__author-name').text() + `|` + $('.album__label-title').text();
+    const articledata = await Promise.all(
+        list.map(async (item) => {
+            const link = $(item).attr('data-link').replace('http://', 'https://');
+            const title = $(item).attr('data-title');
+            const single = {
+                title,
+                link,
+                guid: link,
+            };
+            return await finishArticleItem(ctx, single);
+        })
+    );
+    ctx.state.data = {
+        title: mptitle,
+        link: `https://mp.weixin.qq.com/mp/appmsgalbum?__biz=${biz}&action=getalbum${aidurl}`,
+        item: list.map((item, index) => ({
+            title: articledata[index].title,
+            description: $(item).find('.album__item-img').html() + `<br><br>${articledata[index].description}`,
+            link: articledata[index].link,
+            guid: articledata[index].guid,
+            author: articledata[index].author,
+            pubDate: dayjs.unix($(item).find('.js_article_create_time').text()).format(),
+        })),
+    };
+};

package/lib/v2/wechat/radar.js

@@ -1,12 +1,87 @@
 module.exports = {
+    'careerengine.us': {
+        _name: '微信',
+        posts: [
+            {
+                title: '公众号（CareerEngine 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/author/*id/posts'],
+                target: (params) => `/wechat/ce/${params.id}`,
+            },
+        ],
+    },
+    'cimidata.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（二十次幂来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/a/:id'],
+                target: '/wechat/ce/:id',
+            },
+        ],
+    },
+    'data258.com': {
+        _name: '微信',
+        mp: [
+            {
+                title: '公众号（微阅读来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/', '/article/category/:id'],
+                target: '/wechat/data258/:id?',
+            },
+        ],
+    },
     'feeddd.org': {
         _name: '微信',
         '.': [
             {
-                title: '公众号 (feeddd 来源)',
+                title: '公众号（feeddd 来源）',
                 docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
                 source: ['/'],
             },
         ],
     },
+    'mp.weixin.qq.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众平台系统公告栏目',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/cgi-bin/announce'],
+                target: '/wechat/announce',
+            },
+        ],
+    },
+    'privacyhide.com': {
+        _name: '微信',
+        wechat: [
+            {
+                title: '公众号（wechat-feeds 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+            },
+        ],
+    },
+    'wemp.app': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（wemp.app 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/accounts/:id'],
+                target: '/wechat/wemp/:id',
+            },
+        ],
+    },
+    'wxnmh.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（wxnmh.com 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/:id'],
+                target: (params) => `/wechat/wxnmh/${params.id.replace('user-', '').replace('.htm', '')}`,
+            },
+        ],
+    },
 };

package/lib/v2/wechat/router.js

@@ -1,3 +1,14 @@
 module.exports = function (router) {
+    router.get('/announce', require('./announce'));
+    router.get('/ce/:id', require('./ce'));
+    router.get('/data258/:id?', require('./data258'));
+    router.get('/ershicimi/:id', require('./ershcimi'));
     router.get('/feeddd/:id', require('./feeddd'));
+    router.get('/feeds/:id', require('./feeds'));
+    router.get('/mp/homepage/:biz/:hid/:cid?', require('./mp'));
+    router.get('/mp/msgalbum/:biz/:aid', require('./msgalbum'));
+    router.get('/tgchannel/:id/:mpName?/:searchQueryType?', require('./tgchannel'));
+    router.get('/uread/:userid', require('./uread'));
+    router.get('/wemp/:id', require('./wemp'));
+    router.get('/wxnmh/:id', require('./wxnmh'));
 };