rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/v2/ustc/radar.js

@@ -0,0 +1,53 @@
+module.exports = {
+    'ustc.edu.cn': {
+        _name: '中国科学技术大学',
+        '.': [
+            {
+                title: '官网通知公告',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/news',
+            },
+        ],
+        'www.teach': [
+            {
+                title: '教务处通知新闻',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/jwc',
+            },
+        ],
+        job: [
+            {
+                title: '就业信息网',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/job',
+            },
+        ],
+        gradschool: [
+            {
+                title: '研究生院',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/gs',
+            },
+        ],
+        sist: [
+            {
+                title: '信息科学技术学院',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/sist',
+            },
+        ],
+        eeis: [
+            {
+                title: '电子工程与信息科学系',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-ke-xue-ji-shu-da-xue',
+                source: '/',
+                target: '/ustc/eeis',
+            },
+        ],
+    },
+};

package/lib/v2/ustc/router.js

@@ -0,0 +1,8 @@
+module.exports = function (router) {
+    router.get('/eeis/:type?', require('./eeis'));
+    router.get('/gs/:type?', require('./gs'));
+    router.get('/job/:category?', require('./job'));
+    router.get('/jwc/:type?', require('./jwc'));
+    router.get('/news/:type?', require('./index'));
+    router.get('/sist/:type?', require('./sist'));
+};

package/lib/v2/ustc/sist.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const map = new Map([
+    ['tzgg', { title: '中国科学技术大学信息科学技术学院 - 通知公告', id: '5142' }],
+    ['zsgz', { title: '中国科学技术大学信息科学技术学院 - 招生工作', id: '5108' }],
+]);
+
+const host = 'https://sist.ustc.edu.cn';
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type ?? 'tzgg';
+    const info = map.get(type);
+    if (!info) {
+        throw 'invalid type';
+    }
+    const id = info.id;
+
+    const response = await got(`${host}/${id}/list.htm`);
+    const $ = cheerio.load(response.data);
+    let items = $('div#wp_news_w12')
+        .find('li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const title = item.find('a').attr('title').trim();
+            const link = item.find('a').attr('href').startsWith('/') ? host + item.find('a').attr('href') : item.find('a').attr('href');
+            const pubDate = timezone(parseDate(item.find('span').text(), 'YYYY-MM-DD'), +8);
+            return {
+                title,
+                pubDate,
+                link,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                let desc = '';
+                try {
+                    const response = await got(item.link);
+                    desc = cheerio.load(response.data)('div.wp_articlecontent').html();
+                    item.description = desc;
+                } catch (err) {
+                    // intranet only contents
+                }
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: info.title,
+        link: `${host}/${id}/list.htm`,
+        item: items,
+    };
+};

package/lib/v2/usts/jwch.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const rootURL = 'http://jwch.usts.edu.cn/index';
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type ?? 'jwdt';
+    const url = `${rootURL}/${type}.htm`;
+    const response = await got(url);
+
+    const $ = cheerio.load(response.data);
+    const title = $('div.mainWrap.cleafix > div > div.right.fr > div.local.fl > h3').text();
+    const list = $('div.list > ul > li')
+        .map((_index, item) => ({
+            title: $(item).find('a').text(),
+            link: new URL($(item).find('a').attr('href'), rootURL).href,
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                let author = null;
+                let pubDate = null;
+                content('div.content-title.fl > i')
+                    .text()
+                    .split('  ')
+                    .forEach((item) => {
+                        if (item.includes('作者：')) {
+                            author = item.split('：')[1];
+                        }
+                        if (item.includes('时间：')) {
+                            pubDate = item.split('：')[1];
+                        }
+                    });
+
+                item.description = content('div#vsb_content').html();
+                item.author = author;
+                item.pubDate = timezone(parseDate(pubDate), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `苏州科技大学 教务处 - ${title}`,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/usts/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/juch/:type?': ['Fatpandac'],
+};

package/lib/v2/usts/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'usts.edu.cn': {
+        _name: '苏州科技大学',
+        jwch: [
+            {
+                title: '教务动态',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/jwdt.htm'],
+                target: '/wsts/jwch/jwdt',
+            },
+            {
+                title: '公告在线',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/ggzx.htm'],
+                target: '/wsts/jwch/ggzx',
+            },
+            {
+                title: '选课通知',
+                docs: 'https://docs.rsshub.app/university.html#su-zhou-ke-ji-da-xue',
+                source: ['/xktz.htm'],
+                target: '/wsts/jwch/xktz',
+            },
+        ],
+    },
+};

package/lib/v2/usts/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/jwch/:type?', require('./jwch'));
+};

package/lib/v2/vimeo/templates/description.art

@@ -1,4 +1,4 @@
-<iframe width="640" height="360" src="https://player.vimeo.com/video{{ videoUrl }}" frameborder="0" webkitAllowFullScreen mozallowfullscreen allowFullScreen></iframe>
-{{ if vdescription }}
-   <p>{{@ vdescription }}</p>
-{{ /if }}
+<iframe width="640" height="360" src="https://player.vimeo.com/video{{ videoUrl }}" frameborder="0" webkitAllowFullScreen mozallowfullscreen allowFullScreen></iframe>
+{{ if vdescription }}
+   <p>{{@ vdescription }}</p>
+{{ /if }}

package/lib/v2/vlive/index.js

@@ -0,0 +1,71 @@
+const got = require('@/utils/got');
+const { JSDOM } = require('jsdom');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const baseUrl = 'https://www.vlive.tv';
+
+module.exports = async (ctx) => {
+    const { channel, board } = ctx.params;
+    const link = `${baseUrl}/channel/${channel}/board/${board}`;
+
+    const response = await got(link);
+    const dom = new JSDOM(response.data, {
+        runScripts: 'dangerously',
+    });
+    const preloadedState = dom.window.__PRELOADED_STATE__;
+    const channelData = preloadedState.channel.channel;
+    const userCountry = preloadedState.common.userCountry;
+    const userLanguage = preloadedState.common.userLanguage;
+
+    const apiResponse = await got(`${baseUrl}/globalv-web/vam-web/post/v1.0/board-${board}/posts`, {
+        searchParams: {
+            appId: '8c6cc7b45d2568fb668be6e05b6e5a3b',
+            fields: 'attachments,author,availableActions,board{boardId,title,boardType,payRequired,includedCountries,excludedCountries},channel{channelName,channelCode},totalCommentCount,contentType,createdAt,emotionCount,excludedCountries,includedCountries,isCommentEnabled,isHiddenFromStar,lastModifierMember,notice,officialVideo,plainBody,postId,postVersion,reservation,starReactions,targetMember,thumbnail,title,url,writtenIn,sharedPosts,originPost,blindType',
+            sortType: 'LATEST',
+            limit: ctx.query.limit ? Number(ctx.query.limit) : 20,
+            gcc: userCountry,
+            locale: userLanguage,
+        },
+        headers: {
+            referer: link,
+        },
+    });
+
+    const items = apiResponse.data.data.map((item) => {
+        let description;
+        switch (item.contentType) {
+            case 'POST':
+                description = art(path.join(__dirname, 'templates/post.art'), {
+                    post: item.plainBody?.replace(/\r\n/g, '<br><br>').replace(/\n/g, '<br>'),
+                    images: item.attachments.photo,
+                });
+                break;
+            case 'VIDEO':
+                description = art(path.join(__dirname, 'templates/video.art'), {
+                    link: item.url,
+                    image: item.officialVideo.thumb,
+                });
+                break;
+            default:
+                throw `Unknown content type: ${item.contentType}`;
+        }
+
+        return {
+            title: item.title,
+            description,
+            link: item.url,
+            author: item.author.nickname,
+            pubDate: parseDate(item.createdAt),
+        };
+    });
+
+    ctx.state.data = {
+        title: `${channelData.channelName} Channel`,
+        description: channelData.channelDescription,
+        link,
+        image: channelData.channelProfileImage,
+        item: items,
+    };
+};

package/lib/v2/vlive/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/channel/:channel/board/:board': ['TonyRL'],
+};

package/lib/v2/vlive/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'vlive.tv': {
+        _name: 'V LIVE',
+        '.': [
+            {
+                title: 'Board',
+                docs: 'https://docs.rsshub.app/en/live.html#v-live',
+                source: '/channel/:board/board/:board',
+                target: '/vlive/channel/:board/board/:board',
+            },
+        ],
+    },
+};

package/lib/v2/vlive/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/channel/:channel/board/:board', require('./index'));
+};

package/lib/v2/vlive/templates/post.art

@@ -0,0 +1,8 @@
+{{ if post }}
+{{@ post }}
+{{ /if }}
+{{ if images }}
+{{ each images image }}
+<img src="{{ image.url }}">
+{{ /each }}
+{{ /if }}

package/lib/v2/vlive/templates/video.art

@@ -0,0 +1,3 @@
+{{ if image }}
+<a href="{{ link }}"><img src="{{ image }}"></a>
+{{ /if }}

package/lib/v2/wallpaperhub/index.js

@@ -0,0 +1,28 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const link = 'https://wallpaperhub.app/api/v1/wallpapers/?limit=20&page=&query=&width=&height=&tags=';
+    const response = await got({
+        method: 'get',
+        url: link,
+    });
+
+    const list = response.data.entities.map((item) => ({
+        title: item.entity.title,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            description: item.entity.description,
+            img: item.entity.variations[0].resolutions[0].url || item.entity.thumbnail,
+        }),
+        pubDate: parseDate(item.entity.created),
+        link: `https://wallpaperhub.app/wallpapers/${item.entity.id}`,
+    }));
+
+    ctx.state.data = {
+        title: 'WallpaperHub',
+        link: 'https://wallpaperhub.app/wallpapers',
+        item: list,
+    };
+};

package/lib/v2/wallpaperhub/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['nczitzk'],
+};

package/lib/v2/wallpaperhub/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'wallpaperhub.app': {
+        _name: 'WallpaperHub',
+        '.': [
+            {
+                title: '壁纸',
+                docs: 'https://docs.rsshub.app/picture.html#wallpaperhub',
+                source: ['/wallpaperhub', '/'],
+                target: '/wallpaperhub',
+            },
+        ],
+    },
+};

package/lib/v2/wallpaperhub/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/', require('./index'));
+};

package/lib/v2/wallpaperhub/templates/description.art

@@ -0,0 +1 @@
+<p>{{ description }}</p><img src="{{ img }}">

package/lib/v2/wechat/_README

@@ -0,0 +1 @@
+Make sure you read lib/utils/wechat-mp.js before adding a new route.

package/lib/v2/wechat/announce.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const { data: htmlString } = await got({
+        method: 'get',
+        url: 'https://mp.weixin.qq.com/cgi-bin/announce?action=getannouncementlist&lang=zh_CN',
+    });
+
+    const $ = cheerio.load(htmlString);
+    const announceList = [];
+
+    $('.mp_news_list > .mp_news_item').each(function () {
+        const $item = $(this);
+        const $link = $item.find('a');
+        const time = $item.find('.read_more').text();
+        const title = $item.find('strong').text();
+
+        announceList.push({
+            title: `${time} ${title}`,
+            link: `https://mp.weixin.qq.com${$link.attr('href')}`,
+            description: title,
+            pubDate: parseDate(time),
+        });
+    });
+
+    ctx.state.data = {
+        title: '微信公众平台-系统公告栏目',
+        link: 'https://mp.weixin.qq.com/cgi-bin/announce?action=getannouncementlist&lang=zh_CN',
+        item: announceList,
+    };
+};

package/lib/v2/wechat/ce.js

@@ -0,0 +1,69 @@
+const parser = require('@/utils/rss-parser');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { fixArticleContent } = require('@/utils/wechat-mp');
+const { parseDate } = require('@/utils/parse-date');
+
+// any UA containing "RSS" can pass the check
+// mark the UA as a desktop UA with "(X11; Linux x86_64)"
+const UA = 'Mozilla/5.0 (X11; Linux x86_64) RSS Reader';
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+
+    const feed = await parser.parseString(
+        await got
+            .get(`https://posts.careerengine.us/author/${id}/rss`, {
+                headers: {
+                    'User-Agent': UA,
+                },
+            })
+            .then((_) => _.data)
+    );
+
+    const items = await Promise.all(
+        feed.items.splice(0, 10).map(async (item) => {
+            // generally speaking, changing `item.link` of an existing route could potentially break `item.guid`
+            // but since the route has been down for at least 8 months, it's probably safe
+            item.link = item.link.replace(/^http:\/\//, 'https://');
+            return await ctx.cache.tryGet(item.link, async () => {
+                const response = await got.get(item.link, {
+                    headers: {
+                        'User-Agent': UA,
+                    },
+                });
+
+                const $ = cheerio.load(response.data);
+
+                const description = fixArticleContent($('.post'));
+
+                let pubDate = item.pubDate;
+                if (!pubDate || pubDate === 'Invalid Date') {
+                    // sometimes the pubDate is not available in the official feed
+                    const postDate = $('.post-date')
+                        .text()
+                        .replace(/\s+|发表/g, '');
+                    // the date format is "发表 YYYY年MM月DD日 "
+                    // following the official feed behavior: imprecise date is in UTC
+                    // `<pubDate>Mon, 04 Apr 2022 00:00:00 GMT</pubDate>`
+                    pubDate = parseDate(postDate, 'YYYY年MM月DD日');
+                    pubDate = new Date(pubDate.getTime() - pubDate.getTimezoneOffset() * 60 * 1000);
+                }
+
+                return {
+                    title: item.title,
+                    description,
+                    pubDate,
+                    link: item.link,
+                };
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: `微信公众号 - ${feed.title}`,
+        link: `https://posts.careerengine.us/author/${id}/posts`,
+        description: feed.description,
+        item: items,
+    };
+};

package/lib/v2/wechat/data258.js

@@ -0,0 +1,137 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+const { RequestInProgressError } = require('@/errors');
+const wait = require('@/utils/wait');
+
+const parsePage = ($item, hyperlinkSelector, timeSelector) => {
+    const hyperlink = $item.find(hyperlinkSelector);
+    const title = hyperlink.text();
+    const link = hyperlink.attr('href');
+    const pubDate = timezone(parseDate($item.find(timeSelector).text(), 'YYYY-MM-DD HH:mm'), 8);
+    return {
+        title,
+        link,
+        pubDate,
+    };
+};
+
+module.exports = async (ctx) => {
+    // !!! here we must use a lock to prevent other requests to break the anti-anti-crawler workarounds !!!
+    if ((await ctx.cache.get('data258:lock', false)) === '1') {
+        throw new RequestInProgressError('Another request is in progress, please try again later.');
+    }
+    // !!! here no need to acquire the lock, because the MP/category page has no crawler detection !!!
+
+    const id = ctx.params.id;
+
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 5;
+
+    const rootUrl = 'https://mp.data258.com';
+    const pageUrl = id ? `${rootUrl}/article/category/${id}` : rootUrl;
+
+    const response = await got(pageUrl);
+    const $ = cheerio.load(response.data);
+
+    const title = $('head title').text();
+    // title = title.endsWith('-微阅读') ? title.slice(0, title.length - 4) : title;
+    const description = $('meta[name="description"]').attr('content');
+
+    const categoryPage = $('ul.fly-list');
+
+    let items;
+    if (categoryPage && categoryPage.length) {
+        // got a category page
+        items = $(categoryPage)
+            .find('li')
+            .map((_, item) => parsePage($(item), 'h2 a', '.fly-list-info span'))
+            .get();
+    } else {
+        // got an MP page
+        items = $('ul.jie-row li')
+            .map((_, item) => parsePage($(item), 'a.jie-title', '.layui-hide-xs'))
+            .get();
+    }
+
+    items = items.slice(0, limit); // limit to avoid being anti-crawled
+
+    // !!! double-check !!!
+    if ((await ctx.cache.get('data258:lock', false)) === '1') {
+        throw new RequestInProgressError('Another request is in progress, please try again later.');
+    } else {
+        // !!! here we acquire the lock because the jump page has crawler detection !!!
+        await ctx.cache.set('data258:lock', '1', 60);
+    }
+
+    // !!! here we must use a for-loop to ensure the concurrency is 1 !!!
+    // !!! please do note that if you try to increase the concurrency, your IP will be banned for a long time !!!
+
+    let err; // !!! let RSSHub throw an anti-crawler prompt if the route is empty !!!
+
+    /* eslint-disable no-await-in-loop */
+    for (const item of items) {
+        // https://mp.data258.com/wx?id=${id}&t={token}, id is a permanent hex, token is a temporary base64
+        const cacheId = item.link.match(/id=([\da-f]+)/)[1];
+        item.link = item.link.startsWith('http') ? item.link : `${rootUrl}${item.link}`;
+        const realLink = await ctx.cache.tryGet(`data258:${cacheId}`, async () => {
+            try {
+                // !!! here we must sleep 1s to avoid being anti-crawled !!!
+                // !!! please do note that if the interval is less than 1s, your IP will be banned for a long time !!!
+                await wait(1000);
+
+                const response = await got.get(item.link, {
+                    headers: {
+                        Referer: pageUrl, // essential
+                    },
+                });
+                if (response.data.includes('今日浏览次数已达上限')) {
+                    // !!! as long as cache hits, the link will not be crawled and consume the limit !!!
+                    // !!! so that's not a big problem if the RSSHub instance is self-hosted !!!
+                    err = new got.RequestError(response.data, {}, response.request);
+                    return null;
+                }
+                const $ = cheerio.load(response.data);
+                const jmpJS = $('script')
+                    .filter((_, e) => $(e).html().includes('location.href'))
+                    .html();
+                return jmpJS.match(/location\.href='([^']+)'/)[1];
+            } catch (e) {
+                err = e;
+                return null;
+            }
+        });
+        if (realLink) {
+            item.link = realLink;
+        } else {
+            break; // being anti-crawled, immediately cancel following operations
+        }
+    }
+    /* eslint-enable no-await-in-loop */
+
+    // !!! release the lock, let it expire immediately since no need to keep it in cache !!!
+    await ctx.cache.set('data258:lock', '0', 1);
+
+    // jump links are valid only for a short period of time, drop those un-jumped items
+    // http://mp.weixin.qq.com/s
+    items = items.filter((item) => item.link.match(/^https?:\/\/mp\.weixin\.qq\.com\/s/));
+
+    if (items.length === 0 && err) {
+        // !!! if each request is anti-crawled, the filtered items array will be empty !!!
+        // !!! let RSSHub throw an anti-crawler prompt !!!
+        throw err;
+    }
+
+    await Promise.all(items.map(async (item) => await finishArticleItem(ctx, item, !!categoryPage)));
+
+    ctx.state.data = {
+        title,
+        link: pageUrl,
+        description,
+        item: items,
+    };
+};
+
+// TODO: login? the valid time for cookies seems to be short, and abusing account will probably get banned...
+// TODO: fetch full article for the official RSS feed? unless someone who is VIP contributes their RSS feed for test...