rsshub 1.0.0-master.fb2a407 → 1.0.0-master.fc5a2f6

package/lib/{routes/universities/ustc/jwc/index.js → v2/ustc/jwc.js}

@@ -1,52 +1,49 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+// const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'windows', device: 'desktop' });
 const noticeUrl = 'https://www.teach.ustc.edu.cn/category/notice';
 const noticeType = { teaching: '教学', info: '信息', exam: '考试', exchange: '交流' };
 
 module.exports = async (ctx) => {
-    const type = ctx.params.type || '';
+    const type = ctx.params.type ?? '';
     // 发起 HTTP GET 请求
     const response = await got({
         method: 'get',
-        headers: {
-            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.121 Safari/537.36',
-        },
+
+        /* headers: {
+            'user-agent': UA,
+        }, */
         url: `${noticeUrl}${type === '' ? '' : '-' + type}`,
     });
 
     const $ = cheerio.load(response.data);
-    const list = $(type === '' ? 'ul[class="article-list with-tag"] > li' : 'ul[class=article-list] > li')
+    let items = $(type === '' ? 'ul[class="article-list with-tag"] > li' : 'ul[class=article-list] > li')
         .map(function () {
             const child = $(this).children();
             const info = {
                 title: type === '' ? $(child[0]).find('a').text() + ' - ' + $(child[1]).find('a').text() : $(child[0]).find('a').text(),
                 link: type === '' ? $(child[1]).find('a').attr('href') : $(child[0]).find('a').attr('href'),
+                pubDate: timezone(parseDate($(this).find('.date').text().trim(), 'YYYY-MM-DD'), +8),
             };
             return info;
         })
         .get();
 
-    const out = await Promise.all(
-        list
+    items = await Promise.all(
+        items
             .filter((item) => item.link)
-            .map(async (item) => {
-                const itemUrl = item.link;
-                const cache = await ctx.cache.get(itemUrl);
-                if (cache) {
-                    return Promise.resolve(JSON.parse(cache));
-                }
-                const response = await got.get(itemUrl);
-                const $ = cheerio.load(response.data);
-                const single = {
-                    title: item.title,
-                    link: itemUrl,
-                    description: $('main[class=single]').html(),
-                    pubDate: new Date($('li[class=meta-date]').text()).toUTCString(),
-                };
-                ctx.cache.set(itemUrl, JSON.stringify(single));
-                return Promise.resolve(single);
-            })
+            .map((item) =>
+                ctx.cache.tryGet(item.link, async () => {
+                    const response = await got(item.link);
+                    const $ = cheerio.load(response.data);
+                    // www.teach ?? pms.cmet ?? news
+                    item.description = $('main[class=single]').html() ?? $('.card-footer').html() ?? $('.v_news_content').html();
+                    item.pubDate = $('li[class=meta-date]').text() ? timezone(parseDate($('li[class=meta-date]').text(), 'YYYY-MM-DD HH:mm'), +8) : item.pubDate;
+                    return item;
+                })
+            )
     );
 
     const desc = type === '' ? '中国科学技术大学教务处 - 通知新闻' : `中国科学技术大学教务处 - ${noticeType[type]}类通知`;
@@ -55,6 +52,6 @@ module.exports = async (ctx) => {
         title: desc,
         description: desc,
         link: `${noticeUrl}${type === '' ? '' : '-' + type}`,
-        item: out,
+        item: items,
     };
 };

package/lib/v2/ustc/maintainer.js

@@ -0,0 +1,8 @@
+module.exports = {
+    '/eeis/:type?': ['jasongzy'],
+    '/gs/:type?': ['jasongzy'],
+    '/job/:category?': ['nczitzk'],
+    '/jwc/:type?': ['hang333'],
+    '/news/:type?': ['hang333', 'jasongzy'],
+    '/sist/:type?': ['jasongzy'],
+};

package/lib/v2/ustc/radar.js

@@ -0,0 +1,53 @@
+module.exports = {
+    'ustc.edu.cn': {
+        _name: '中国科学技术大学',
+        '.': [
+            {
+                title: '官网通知公告',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/news',
+            },
+        ],
+        'www.teach': [
+            {
+                title: '教务处通知新闻',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/jwc',
+            },
+        ],
+        job: [
+            {
+                title: '就业信息网',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/job',
+            },
+        ],
+        gradschool: [
+            {
+                title: '研究生院',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/gs',
+            },
+        ],
+        sist: [
+            {
+                title: '信息科学技术学院',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/sist',
+            },
+        ],
+        eeis: [
+            {
+                title: '电子工程与信息科学系',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/eeis',
+            },
+        ],
+    },
+};

package/lib/v2/ustc/router.js

@@ -0,0 +1,8 @@
+module.exports = function (router) {
+    router.get('/eeis/:type?', require('./eeis'));
+    router.get('/gs/:type?', require('./gs'));
+    router.get('/job/:category?', require('./job'));
+    router.get('/jwc/:type?', require('./jwc'));
+    router.get('/news/:type?', require('./index'));
+    router.get('/sist/:type?', require('./sist'));
+};

package/lib/v2/ustc/sist.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const map = new Map([
+    ['tzgg', { title: '中国科学技术大学信息科学技术学院 - 通知公告', id: '5142' }],
+    ['zsgz', { title: '中国科学技术大学信息科学技术学院 - 招生工作', id: '5108' }],
+]);
+
+const host = 'https://sist.ustc.edu.cn';
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type ?? 'tzgg';
+    const info = map.get(type);
+    if (!info) {
+        throw 'invalid type';
+    }
+    const id = info.id;
+
+    const response = await got(`${host}/${id}/list.htm`);
+    const $ = cheerio.load(response.data);
+    let items = $('div#wp_news_w12')
+        .find('li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const title = item.find('a').attr('title').trim();
+            const link = item.find('a').attr('href').startsWith('/') ? host + item.find('a').attr('href') : item.find('a').attr('href');
+            const pubDate = timezone(parseDate(item.find('span').text(), 'YYYY-MM-DD'), +8);
+            return {
+                title,
+                pubDate,
+                link,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                let desc = '';
+                try {
+                    const response = await got(item.link);
+                    desc = cheerio.load(response.data)('div.wp_articlecontent').html();
+                    item.description = desc;
+                } catch (err) {
+                    // intranet only contents
+                }
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: info.title,
+        link: `${host}/${id}/list.htm`,
+        item: items,
+    };
+};

package/lib/v2/usts/jwch.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const rootURL = 'http://jwch.usts.edu.cn/index';
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type ?? 'jwdt';
+    const url = `${rootURL}/${type}.htm`;
+    const response = await got(url);
+
+    const $ = cheerio.load(response.data);
+    const title = $('div.mainWrap.cleafix > div > div.right.fr > div.local.fl > h3').text();
+    const list = $('div.list > ul > li')
+        .map((_index, item) => ({
+            title: $(item).find('a').text(),
+            link: new URL($(item).find('a').attr('href'), rootURL).href,
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                let author = null;
+                let pubDate = null;
+                content('div.content-title.fl > i')
+                    .text()
+                    .split('  ')
+                    .forEach((item) => {
+                        if (item.includes('作者：')) {
+                            author = item.split('：')[1];
+                        }
+                        if (item.includes('时间：')) {
+                            pubDate = item.split('：')[1];
+                        }
+                    });
+
+                item.description = content('div#vsb_content').html();
+                item.author = author;
+                item.pubDate = timezone(parseDate(pubDate), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `苏州科技大学 教务处 - ${title}`,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/usts/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/juch/:type?': ['Fatpandac'],
+};

package/lib/v2/usts/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'usts.edu.cn': {
+        _name: '苏州科技大学',
+        jwch: [
+            {
+                title: '教务动态',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/jwdt.htm'],
+                target: '/wsts/jwch/jwdt',
+            },
+            {
+                title: '公告在线',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/ggzx.htm'],
+                target: '/wsts/jwch/ggzx',
+            },
+            {
+                title: '选课通知',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/xktz.htm'],
+                target: '/wsts/jwch/xktz',
+            },
+        ],
+    },
+};

package/lib/v2/usts/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/jwch/:type?', require('./jwch'));
+};

package/lib/v2/vimeo/templates/description.art

@@ -1,4 +1,4 @@
-<iframe width="640" height="360" src="https://player.vimeo.com/video{{ videoUrl }}" frameborder="0" webkitAllowFullScreen mozallowfullscreen allowFullScreen></iframe>
-{{ if vdescription }}
-   <p>{{@ vdescription }}</p>
-{{ /if }}
+<iframe width="640" height="360" src="https://player.vimeo.com/video{{ videoUrl }}" frameborder="0" webkitAllowFullScreen mozallowfullscreen allowFullScreen></iframe>
+{{ if vdescription }}
+   <p>{{@ vdescription }}</p>
+{{ /if }}

package/lib/v2/wallpaperhub/index.js

@@ -0,0 +1,28 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const link = 'https://wallpaperhub.app/api/v1/wallpapers/?limit=20&page=&query=&width=&height=&tags=';
+    const response = await got({
+        method: 'get',
+        url: link,
+    });
+
+    const list = response.data.entities.map((item) => ({
+        title: item.entity.title,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            description: item.entity.description,
+            img: item.entity.variations[0].resolutions[0].url || item.entity.thumbnail,
+        }),
+        pubDate: parseDate(item.entity.created),
+        link: `https://wallpaperhub.app/wallpapers/${item.entity.id}`,
+    }));
+
+    ctx.state.data = {
+        title: 'WallpaperHub',
+        link: 'https://wallpaperhub.app/wallpapers',
+        item: list,
+    };
+};

package/lib/v2/wallpaperhub/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['nczitzk'],
+};

package/lib/v2/wallpaperhub/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'wallpaperhub.app': {
+        _name: 'WallpaperHub',
+        '.': [
+            {
+                title: '壁纸',
+                docs: 'https://docs.rsshub.app/picture.html#wallpaperhub',
+                source: ['/wallpaperhub', '/'],
+                target: '/wallpaperhub',
+            },
+        ],
+    },
+};

package/lib/v2/wallpaperhub/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/', require('./index'));
+};

package/lib/v2/wallpaperhub/templates/description.art

@@ -0,0 +1 @@
+<p>{{ description }}</p><img src="{{ img }}">

package/lib/{routes/tencent → v2}/wechat/announce.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const { data: htmlString } = await got({
@@ -20,13 +21,13 @@ module.exports = async (ctx) => {
             title: `${time} ${title}`,
             link: `https://mp.weixin.qq.com${$link.attr('href')}`,
             description: title,
-            pubDate: new Date(time).toUTCString(),
+            pubDate: parseDate(time),
         });
     });
 
     ctx.state.data = {
         title: '微信公众平台-系统公告栏目',
-        link: 'https://mp.weixin.qq.com/cgi-bin/announce?action=getannouncementlist&lang=zh_CN&token=',
+        link: 'https://mp.weixin.qq.com/cgi-bin/announce?action=getannouncementlist&lang=zh_CN',
         item: announceList,
     };
 };

package/lib/v2/wechat/data258.js

@@ -0,0 +1,137 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+const { RequestInProgressError } = require('@/errors');
+const wait = require('@/utils/wait');
+
+const parsePage = ($item, hyperlinkSelector, timeSelector) => {
+    const hyperlink = $item.find(hyperlinkSelector);
+    const title = hyperlink.text();
+    const link = hyperlink.attr('href');
+    const pubDate = timezone(parseDate($item.find(timeSelector).text(), 'YYYY-MM-DD HH:mm'), 8);
+    return {
+        title,
+        link,
+        pubDate,
+    };
+};
+
+module.exports = async (ctx) => {
+    // !!! here we must use a lock to prevent other requests to break the anti-anti-crawler workarounds !!!
+    if ((await ctx.cache.get('data258:lock', false)) === '1') {
+        throw new RequestInProgressError('Another request is in progress, please try again later.');
+    }
+    // !!! here no need to acquire the lock, because the MP/category page has no crawler detection !!!
+
+    const id = ctx.params.id;
+
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 5;
+
+    const rootUrl = 'https://mp.data258.com';
+    const pageUrl = id ? `${rootUrl}/article/category/${id}` : rootUrl;
+
+    const response = await got(pageUrl);
+    const $ = cheerio.load(response.data);
+
+    const title = $('head title').text();
+    // title = title.endsWith('-微阅读') ? title.slice(0, title.length - 4) : title;
+    const description = $('meta[name="description"]').attr('content');
+
+    const categoryPage = $('ul.fly-list');
+
+    let items;
+    if (categoryPage && categoryPage.length) {
+        // got a category page
+        items = $(categoryPage)
+            .find('li')
+            .map((_, item) => parsePage($(item), 'h2 a', '.fly-list-info span'))
+            .get();
+    } else {
+        // got an MP page
+        items = $('ul.jie-row li')
+            .map((_, item) => parsePage($(item), 'a.jie-title', '.layui-hide-xs'))
+            .get();
+    }
+
+    items = items.slice(0, limit); // limit to avoid being anti-crawled
+
+    // !!! double-check !!!
+    if ((await ctx.cache.get('data258:lock', false)) === '1') {
+        throw new RequestInProgressError('Another request is in progress, please try again later.');
+    } else {
+        // !!! here we acquire the lock because the jump page has crawler detection !!!
+        await ctx.cache.set('data258:lock', '1', 60);
+    }
+
+    // !!! here we must use a for-loop to ensure the concurrency is 1 !!!
+    // !!! please do note that if you try to increase the concurrency, your IP will be banned for a long time !!!
+
+    let err; // !!! let RSSHub throw an anti-crawler prompt if the route is empty !!!
+
+    /* eslint-disable no-await-in-loop */
+    for (const item of items) {
+        // https://mp.data258.com/wx?id=${id}&t={token}, id is a permanent hex, token is a temporary base64
+        const cacheId = item.link.match(/id=([\da-f]+)/)[1];
+        item.link = item.link.startsWith('http') ? item.link : `${rootUrl}${item.link}`;
+        const realLink = await ctx.cache.tryGet(`data258:${cacheId}`, async () => {
+            try {
+                // !!! here we must sleep 1s to avoid being anti-crawled !!!
+                // !!! please do note that if the interval is less than 1s, your IP will be banned for a long time !!!
+                await wait(1000);
+
+                const response = await got.get(item.link, {
+                    headers: {
+                        Referer: pageUrl, // essential
+                    },
+                });
+                if (response.data.includes('今日浏览次数已达上限')) {
+                    // !!! as long as cache hits, the link will not be crawled and consume the limit !!!
+                    // !!! so that's not a big problem if the RSSHub instance is self-hosted !!!
+                    err = new got.RequestError(response.data, {}, response.request);
+                    return null;
+                }
+                const $ = cheerio.load(response.data);
+                const jmpJS = $('script')
+                    .filter((_, e) => $(e).html().includes('location.href'))
+                    .html();
+                return jmpJS.match(/location\.href='([^']+)'/)[1];
+            } catch (e) {
+                err = e;
+                return null;
+            }
+        });
+        if (realLink) {
+            item.link = realLink;
+        } else {
+            break; // being anti-crawled, immediately cancel following operations
+        }
+    }
+    /* eslint-enable no-await-in-loop */
+
+    // !!! release the lock, let it expire immediately since no need to keep it in cache !!!
+    await ctx.cache.set('data258:lock', '0', 1);
+
+    // jump links are valid only for a short period of time, drop those un-jumped items
+    // http://mp.weixin.qq.com/s
+    items = items.filter((item) => item.link.match(/^https?:\/\/mp\.weixin\.qq\.com\/s/));
+
+    if (items.length === 0 && err) {
+        // !!! if each request is anti-crawled, the filtered items array will be empty !!!
+        // !!! let RSSHub throw an anti-crawler prompt !!!
+        throw err;
+    }
+
+    await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item, !!categoryPage)));
+
+    ctx.state.data = {
+        title,
+        link: pageUrl,
+        description,
+        item: items,
+    };
+};
+
+// TODO: login? the valid time for cookies seems to be short, and abusing account will probably get banned...
+// TODO: fetch full article for the official RSS feed? unless someone who is VIP contributes their RSS feed for test...

package/lib/v2/wechat/maintainer.js

@@ -1,3 +1,14 @@
 module.exports = {
+    '/announce': ['xyqfer'],
+    '/ce/:id': ['HenryQW'],
+    '/ershicimi/:id': ['sanmmm'],
+    '/data258/:id?': ['Rongronggg9'],
     '/feeddd/:id': ['TonyRL', 'Rongronggg9'],
+    '/feeds': ['tylinux'],
+    '/mp/homepage/:biz/:hid/:cid?': ['MisteryMonster'],
+    '/mp/msgalbum/:biz/:aid': ['MisteryMonster'],
+    '/tgchannel/:id/:mpName?/:searchQueryType?': ['LogicJake', 'Rongronggg9'],
+    '/uread/:userid': ['kt286'],
+    '/wemp/:id': ['HenryQW'],
+    '/wxnmh/:id': ['laampui'],
 };

package/lib/v2/wechat/radar.js

@@ -1,12 +1,87 @@
 module.exports = {
+    'careerengine.us': {
+        _name: '微信',
+        posts: [
+            {
+                title: '公众号（CareerEngine 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/author/*id/posts'],
+                target: (params) => `/wechat/ce/${params.id}`,
+            },
+        ],
+    },
+    'cimidata.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（二十次幂来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/a/:id'],
+                target: '/wechat/ce/:id',
+            },
+        ],
+    },
+    'data258.com': {
+        _name: '微信',
+        mp: [
+            {
+                title: '公众号（微阅读来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/', '/article/category/:id'],
+                target: '/wechat/data258/:id?',
+            },
+        ],
+    },
     'feeddd.org': {
         _name: '微信',
         '.': [
             {
-                title: '公众号 (feeddd 来源)',
+                title: '公众号（feeddd 来源）',
                 docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
                 source: ['/'],
             },
         ],
     },
+    'mp.weixin.qq.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众平台系统公告栏目',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/cgi-bin/announce'],
+                target: '/wechat/announce',
+            },
+        ],
+    },
+    'privacyhide.com': {
+        _name: '微信',
+        wechat: [
+            {
+                title: '公众号（wechat-feeds 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+            },
+        ],
+    },
+    'wemp.app': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（wemp.app 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/accounts/:id'],
+                target: '/wechat/wemp/:id',
+            },
+        ],
+    },
+    'wxnmh.com': {
+        _name: '微信',
+        '.': [
+            {
+                title: '公众号（wxnmh.com 来源）',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xin',
+                source: ['/:id'],
+                target: (params) => `/wechat/wxnmh/${params.id.replace('user-', '').replace('.htm', '')}`,
+            },
+        ],
+    },
 };