rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/routes/universities/nuaa/cs/index.js

@@ -1,84 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-const getCookie = require('../utils/pypasswaf');
-const host = 'http://cs.nuaa.edu.cn/';
-
-const map = new Map([
-    ['tzgg', { title: '南京航空航天大学计算机科学与技术学院 -- 通知公告', suffix: '1995/list.htm' }],
-    ['xwdt', { title: '南京航空航天大学计算机科学与技术学院 -- 新闻动态', suffix: '1997/list.htm' }],
-    ['kydt', { title: '南京航空航天大学计算机科学与技术学院 -- 科研动态', suffix: '1975/list.htm' }],
-    ['jxdt', { title: '南京航空航天大学计算机科学与技术学院 -- 教学动态', suffix: '1977/list.htm' }],
-    ['xsgz', { title: '南京航空航天大学计算机科学与技术学院 -- 学生工作', suffix: '1959/list.htm' }],
-    ['zsxx', { title: '南京航空航天大学计算机科学与技术学院 -- 招生信息', suffix: '1993/list.htm' }],
-    ['jyxx', { title: '南京航空航天大学计算机科学与技术学院 -- 就业信息', suffix: '1994/list.htm' }],
-]);
-
-module.exports = async (ctx) => {
-    const type = ctx.params.type || 'tzgg';
-    const suffix = map.get(type).suffix;
-
-    const link = url.resolve(host, suffix);
-    const cookie = await getCookie();
-    const gotConfig = {
-        headers: {
-            cookie,
-        },
-    };
-    const response = await got.get(link, gotConfig);
-    const $ = cheerio.load(response.data);
-
-    const list = $('#news_list > ul > li')
-        .slice(0, 10)
-        .map(function () {
-            const info = {
-                title: $(this).find('a').attr('title'),
-                link: $(this).find('a').attr('href'),
-                date: $(this).find('span').text(),
-            };
-            return info;
-        })
-        .get();
-
-    const out = await Promise.all(
-        list.map(async (info) => {
-            const title = info.title || 'tzgg';
-            const date = info.date;
-            const itemUrl = url.resolve(host, info.link);
-
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const arr = itemUrl.split('.');
-            const pageType = arr[arr.length - 1];
-
-            let description = itemUrl;
-            if (pageType === 'htm' || pageType === 'html') {
-                const response = await got.get(itemUrl, gotConfig);
-                const $ = cheerio.load(response.data);
-                description = $('.wp_articlecontent')
-                    .html()
-                    .replace(/src="\//g, `src="${url.resolve(host, '.')}`)
-                    .trim();
-            }
-
-            const single = {
-                title,
-                link: itemUrl,
-                description,
-                pubDate: new Date(date).toUTCString(),
-            };
-            ctx.cache.set(itemUrl, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
-    );
-
-    ctx.state.data = {
-        title: map.get(type).title,
-        link,
-        description: '南京航空航天大学计算机科学与技术学院RSS',
-        item: out,
-    };
-};

package/lib/routes/universities/nuaa/jwc/jwc.js

@@ -1,105 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-const host = 'http://aao.nuaa.edu.cn/';
-// https is not allowed in nuaa;interesting ha
-
-const map = {
-    default: 8230,
-    jxfw: 8230, // 教学服务
-    xspy: 8231, // 学生培养
-    jxjs: 8232, // 教学建设
-    jxzy: 8233, // 教学资源
-};
-
-async function load(link, cookie, ctx) {
-    const cache = await ctx.cache.get(link);
-    if (cache) {
-        return JSON.parse(cache);
-    }
-    const response = await got.get(link, {
-        headers: {
-            cookie,
-        },
-    });
-    const $ = cheerio.load(response.data);
-    const pubDate = new Date(
-        $('.release-time')
-            .text()
-            .slice(-10)
-            .match(/\d{4}-\d{2}-\d{2}/)
-    ).toUTCString();
-    const images = $('img');
-    for (let k = 0; k < images.length; k++) {
-        $(images[k]).replaceWith(`<img src="${url.resolve(host, $(images[k]).attr('src'))}" />`);
-    }
-    const description = $('.wp_articlecontent').html();
-    const result = { pubDate, description };
-    ctx.cache.set(link, JSON.stringify(result));
-
-    return result;
-}
-
-module.exports = async (ctx) => {
-    const browser = await require('@/utils/puppeteer')();
-    const type = ctx.params.type || 'default';
-    const listUrl = `${host}${map[type]}/list.htm`;
-
-    const page = await browser.newPage();
-    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36');
-    await page.evaluateOnNewDocument(() => {
-        // eslint-disable-next-line
-        Object.defineProperty(navigator, 'webdriver', { get: () => false });
-    });
-    // 南航的新waf现在采用cookie验证的方法防止爬虫，并且会对chrome headless进行检测
-
-    await page.goto(listUrl, {
-        waitUntil: 'networkidle0',
-    });
-
-    let msg;
-    try {
-        // data = await page.$('ul.right-ul')
-        msg = await page.$$eval('ul.right-ul > li', (es) =>
-            es.map((e) => {
-                const html = e.innerHTML;
-
-                let [, title] = html.match(/<a.+?>(.+)<\/a>/);
-                if (title.match(/font/)) {
-                    [, title] = title.match(/>(.+)</);
-                }
-                const [, href] = html.match(/href="(.+?)"/);
-                // const [, time] = html.match(/-time">(.+?)</);
-                return {
-                    title,
-                    link: href,
-                    guid: href,
-                };
-            })
-        );
-    } catch (e) {
-        // debugger
-    }
-
-    let cookie = await page.cookies();
-    cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; ');
-    browser.close();
-    msg = await Promise.all(
-        msg.map(async (e) => {
-            const link = url.resolve(host, e.link);
-            return {
-                ...e,
-                link,
-                guid: link,
-                ...(await load(link, cookie, ctx)),
-            };
-        })
-    );
-
-    ctx.state.data = {
-        title: '南航教务',
-        link: host,
-        description: '南航教务RSS',
-        item: msg,
-    };
-};

package/lib/routes/universities/nuaa/utils/pypasswaf.js

@@ -1,24 +0,0 @@
-const host = 'http://aao.nuaa.edu.cn/';
-
-/**
- * async function 获取cookie
- * @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.header.cookie`即可
- */
-module.exports = async function getCookie() {
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36');
-    await page.evaluateOnNewDocument(() => {
-        // eslint-disable-next-line
-        Object.defineProperty(navigator, 'webdriver', { get: () => false });
-    });
-
-    await page.goto(host, {
-        waitUntil: 'networkidle0',
-    });
-
-    let cookie = await page.cookies();
-    cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; ');
-    await browser.close();
-    return cookie;
-};

package/lib/routes/universities/nuaa/yjsy/yjsy.js

@@ -1,61 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-const host = 'http://www.graduate.nuaa.edu.cn/';
-const formatPubDate = require('@/utils/date.js');
-
-const map = {
-    latest: '2146/list.htm',
-    yyxw: 'yyxw/list.htm',
-    sjwj: 'sjwj/list.htm',
-    glwj: '2017/list.htm',
-    xxfw: '2147/list.htm',
-};
-
-async function load(link, ctx) {
-    const cache = await ctx.cache.get(link);
-    if (cache) {
-        return cache;
-    }
-    const response = await got.get(host + link);
-    const $ = cheerio.load(response.data);
-    const images = $('img');
-    for (let k = 0; k < images.length; k++) {
-        $(images[k]).replaceWith(`<img src="${url.resolve(host, $(images[k]).attr('src'))}" />`);
-    }
-    const description = $('.wp_articlecontent').html();
-    ctx.cache.set(link, description);
-    return { description };
-}
-
-module.exports = async (ctx) => {
-    const type = ctx.params.type || 'latest';
-    const response = await got({
-        method: 'get',
-        url: host + map[type],
-    });
-    const $ = cheerio.load(response.data);
-    const list = $('#news_list ul.news_ul > li').slice(0, 10).get();
-    const process = await Promise.all(
-        list.map(async (item) => {
-            const a = $(item).find('a');
-            const t = $(item).find('span');
-            const itemUrl = a.attr('href');
-            const single = {
-                title: a.text(),
-                link: url.resolve(host, itemUrl),
-                guid: url.resolve(host, itemUrl),
-                pubDate: formatPubDate(t.text()),
-            };
-            const other = await load(itemUrl, ctx);
-            return Promise.resolve(Object.assign({}, single, other));
-        })
-    );
-
-    ctx.state.data = {
-        title: '南航研究生院',
-        link: host,
-        description: '南航研究生院RSS',
-        item: process,
-    };
-};

package/lib/routes/universities/pku/bbs/hot.js

@@ -1,61 +0,0 @@
-const config = require('@/config').value;
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-const resolveUrl = require('url').resolve;
-
-function parseDate(text) {
-    const match = /[0-9 :-]+/.exec(text);
-    if (match) {
-        return new Date(match[0]).toUTCString();
-    }
-}
-
-module.exports = async (ctx) => {
-    const cookie = config.pkubbs.cookie;
-    const headers = {};
-    if (cookie) {
-        headers.cookie = cookie;
-    }
-    const r = await got('https://bbs.pku.edu.cn/v2/hot-topic.php', { headers });
-    const $ = cheerio.load(r.body);
-    const listItems = $('#list-content .list-item')
-        .map(function () {
-            return {
-                url: resolveUrl(r.url, $(this).find('> a.link').attr('href')),
-                title: $(this).find('.title').text(),
-            };
-        })
-        .get()
-        .slice(0, 10);
-
-    const item = await Promise.all(
-        listItems.map(({ url, title }) =>
-            ctx.cache.tryGet(url, async () => {
-                // try catch 处理部分无 Cookie 时无法访问的帖子
-                try {
-                    const r = await got(url, { headers });
-                    const $ = cheerio.load(r.body);
-                    return {
-                        title,
-                        description: $('.post-card:first-child .content').html(),
-                        link: url,
-                        guid: url,
-                        pubDate: parseDate($('.post-card:first-child .sl-triangle-container .title span').text()),
-                    };
-                } catch (error) {
-                    return {
-                        title,
-                        link: url,
-                        guid: url,
-                    };
-                }
-            })
-        )
-    );
-    ctx.state.data = {
-        title: '北大未名BBS 全站十大',
-        link: 'https://bbs.pku.edu.cn/v2/hot-topic.php',
-        description: '北大未名BBS 全站热门话题前十名',
-        item,
-    };
-};

package/lib/routes/universities/pku/cls/lecture.js

@@ -1,27 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-const homeUrl = 'http://www.bio.pku.edu.cn/homes/Index/news_jz/7/7.html';
-const baseUrl = 'http://www.bio.pku.edu.cn';
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: homeUrl,
-    });
-
-    const $ = cheerio.load(response.data);
-    ctx.state.data = {
-        title: `北京大学生命科学学院近期讲座`,
-        link: homeUrl,
-        description: `北京大学生命科学学院近期讲座`,
-        item: $('a.clearfix')
-            .map((index, item) => ({
-                title: $(item).find('p').text().trim(),
-                description: '日期: ' + $(item).find('span'), // ${item.find('.chair_txt div').find('span').second().text()}
-
-                link: baseUrl + $('a.clearfix').attr('href'),
-            }))
-            .get(),
-    };
-};

package/lib/routes/universities/pku/eecs.js

@@ -1,41 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const host = 'http://eecs.pku.edu.cn/';
-
-    let type = ctx.params && ctx.params.type;
-    if (type === undefined) {
-        type = '0';
-    }
-
-    const response = await got({
-        method: 'get',
-        url: host + 'Survey/Notice/?Mtitle=' + type,
-        headers: {
-            Referer: host,
-        },
-    });
-
-    const $ = cheerio.load(response.data);
-    const text = $('.hvr-shutter-out-vertical').slice(0, 10);
-
-    ctx.state.data = {
-        title: '北大信科通知',
-        link: host + 'Survey/Notice/?Mtitle=' + type,
-        description: '北大信科 公告通知',
-        item:
-            text &&
-            text
-                .map((index, item) => {
-                    item = $(item);
-                    return {
-                        title: item.find('p').text(),
-                        link: host + 'Survey/Notice/' + item.attr('href'),
-                        description: item.find('p').text(),
-                        pubDate: item.find('em').text(),
-                    };
-                })
-                .get(),
-    };
-};

package/lib/routes/universities/pku/pkuyjs.js

@@ -1,23 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const link = 'https://admission.pku.edu.cn/zsxx/sszs/index.htm';
-    const response = await got.get(link);
-    const $ = cheerio.load(response.data);
-    const list = $('.zsxx_cont_list li').slice(0, 10);
-
-    ctx.state.data = {
-        title: '北京大学研究生院',
-        link,
-        description: '北京大学研究生院通知公告',
-        item:
-            list &&
-            list
-                .map((index, item) => {
-                    item = $(item);
-                    return { title: item.find('li a').text(), description: item.find('li a').text(), link: item.find('li a').attr('href') };
-                })
-                .get(),
-    };
-};

package/lib/routes/universities/pku/rccp/mzyt.js

@@ -1,33 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-const baseUrl = 'http://www.rccp.pku.edu.cn/mzyt/';
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: baseUrl,
-    });
-
-    const $ = cheerio.load(response.data);
-    ctx.state.data = {
-        title: `每周一推 - 北京大学中国政治学研究中心`,
-        link: baseUrl,
-        description: `北京大学中国政治学研究中心，北大中国政治学研究中心，北大政治学研究中心，中国政治学研究中心，政治学研究中心，政治学，北大政治学，北京大学，俞可平
-        北京大学中国政治学研究中心官方网站：www.rccp.pku.edu.cn 。
-         北京大学中国政治学研究中心微信公众平台：“北大政治学”（微信号：PKURCCP）`,
-        item: $('li.list')
-            .map((index, item) => ({
-                title: $(item).find('a').text().trim(),
-                description: '',
-                pubDate: $(item)
-                    .find('span')
-                    .first()
-                    .text()
-                    .match(/\d{4}-\d{2}-\d{2}/g)[0],
-
-                link: baseUrl + $(item).find('a').attr('href'),
-            }))
-            .get(),
-    };
-};

package/lib/routes/universities/sdu/cmse.js

@@ -1,55 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-
-const host = 'http://www.cmse.sdu.edu.cn/';
-const typelist = ['通知公告', '学院新闻', '本科生教育', '研究生教育', '学术动态'];
-const urlList = ['zxzx/tzgg.htm', 'zxzx/xyxw.htm', 'zxzx/bksjy.htm', 'zxzx/yjsjy.htm', 'zxzx/xsdt.htm'];
-
-module.exports = async (ctx) => {
-    const type = parseInt(ctx.params.type) || 0;
-    const link = url.resolve(host, urlList[type]);
-    const response = await got.get(link);
-
-    const $ = cheerio.load(response.data);
-
-    const list = $('#list_right_list a')
-        .slice(0, 10)
-        .map((i, e) => $(e).attr('href'))
-        .get();
-
-    const out = await Promise.all(
-        list
-            .filter((e) => e.startsWith('../info'))
-            .map(async (itemUrl) => {
-                itemUrl = url.resolve(host, itemUrl.slice('3'));
-                const cache = await ctx.cache.get(itemUrl);
-                if (cache) {
-                    return Promise.resolve(JSON.parse(cache));
-                }
-
-                const response = await got.get(itemUrl);
-                const $ = cheerio.load(response.data);
-
-                const rawDate = $('#show_info').text().split(/\s{4}/);
-                let date = rawDate[0].split('：')[1];
-                date = date.slice(0, 4) + '-' + date.slice(5, 7) + '-' + date.slice(8, 10) + ' ' + date.slice(12);
-
-                const single = {
-                    title: $('#show_title').text().trim(),
-                    link: itemUrl,
-                    author: '山东大学材料科学与工程学院',
-                    description: $('#show_content').html(),
-                    pubDate: new Date(date),
-                };
-                ctx.cache.set(itemUrl, JSON.stringify(single));
-                return Promise.resolve(single);
-            })
-    );
-
-    ctx.state.data = {
-        title: `山东大学材料科学与工程学院${typelist[type]}`,
-        link,
-        item: out,
-    };
-};

package/lib/routes/universities/sdu/cs.js

@@ -1,55 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-
-const host = 'http://www.cs.sdu.edu.cn/';
-const typelist = ['学院公告', '学术报告', '新闻动态'];
-const urlList = ['index/xygg.htm', 'xwgg/xsbg.htm', 'xwgg/xyxw.htm'];
-
-module.exports = async (ctx) => {
-    const type = parseInt(ctx.params.type) || 0;
-    const link = url.resolve(host, urlList[type]);
-
-    const response = await got.get(link);
-
-    const $ = cheerio.load(response.data);
-
-    const dateDict = {};
-    const list = $('.sub_text .news-list')
-        .slice(0, 10)
-        .map((i, e) => {
-            const divs = $(e).children();
-            const tlink = 'http://www.cs.sdu.edu.cn/' + $('a', divs[1]).attr('href').substring(3);
-            dateDict[tlink] = new Date($(divs[2]).text()).toUTCString();
-            return tlink;
-        })
-        .get();
-
-    const out = await Promise.all(
-        list.map(async (itemUrl) => {
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const response = await got.get(itemUrl);
-            const $ = cheerio.load(response.data);
-
-            const single = {
-                title: $('#newsTitle').text().trim(),
-                author: '山东大学计算机科学与技术学院',
-                description: $('.v_news_content').html(),
-                pubDate: dateDict[itemUrl],
-                link: itemUrl,
-            };
-            ctx.cache.set(itemUrl, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
-    );
-
-    ctx.state.data = {
-        title: `山东大学计算机科学与技术学院${typelist[type]}通知`,
-        link,
-        item: out,
-    };
-};

package/lib/routes/universities/sdu/epe.js

@@ -1,54 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-
-const host = 'http://www.epe.sdu.edu.cn/';
-const typelist = ['学院动态', '通知公告', '学术论坛'];
-const urlList = ['zxzx/xydt.htm', 'zxzx/tzgg.htm', 'zxzx/xslt.htm'];
-
-module.exports = async (ctx) => {
-    const type = parseInt(ctx.params.type) || 0;
-    const link = url.resolve(host, urlList[type]);
-    const response = await got.get(link);
-
-    const $ = cheerio.load(response.data);
-
-    const list = $('#page_right_main li a')
-        .slice(0, 10)
-        .map((i, e) => $(e).attr('href'))
-        .get();
-
-    const out = await Promise.all(
-        list
-            .filter((e) => e.startsWith('../info'))
-            .map(async (itemUrl) => {
-                itemUrl = url.resolve(host, itemUrl.slice('3'));
-                const cache = await ctx.cache.get(itemUrl);
-                if (cache) {
-                    return Promise.resolve(JSON.parse(cache));
-                }
-
-                const response = await got.get(itemUrl);
-                const $ = cheerio.load(response.data);
-
-                const rawDate = $('#show_info').text().split(/\s{4}/);
-                const date = rawDate[0].split('：')[1];
-
-                const single = {
-                    title: $('#show_title').text().trim(),
-                    link: itemUrl,
-                    author: '山东大学能源与动力工程学院',
-                    description: $('#show_content').html(),
-                    pubDate: new Date(date),
-                };
-                ctx.cache.set(itemUrl, JSON.stringify(single));
-                return Promise.resolve(single);
-            })
-    );
-
-    ctx.state.data = {
-        title: `山东大学能源与动力工程学院${typelist[type]}`,
-        link,
-        item: out,
-    };
-};