rsshub 1.0.0-master.f944cc9 → 1.0.0-master.f97cdb2

package/lib/routes/universities/bjfu/news/utils.js

@@ -1,62 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const iconv = require('iconv-lite'); // 转码
-
-// 完整文章页
-async function load(link) {
-    const response = await got.get(link, {
-        responseType: 'buffer',
-    });
-
-    const data = iconv.decode(response.data, 'gb2312'); // 转码
-
-    // 加载文章内容
-    const $ = cheerio.load(data);
-
-    // 解析日期
-    const date = new Date(
-        $('.article')
-            .text()
-            .match(/\d{4}-\d{2}-\d{2}/)
-    );
-
-    const timeZone = 8;
-    const serverOffset = date.getTimezoneOffset() / 60;
-    const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
-
-    // 提取内容
-    const description = $('.article_con').html();
-
-    // 返回解析的结果
-    return { description, pubDate };
-}
-
-const ProcessFeed = (base, list, caches) =>
-    Promise.all(
-        // 遍历每一篇文章
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-
-            const $title = $('a');
-            // 还原相对链接为绝对链接
-            const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
-
-            // 列表上提取到的信息
-            const single = {
-                title: $title.text(),
-                link: itemUrl,
-                author: '绿色新闻网',
-                guid: itemUrl,
-            };
-
-            // 使用tryGet方法从缓存获取内容。
-            // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
-            const other = await caches.tryGet(itemUrl, () => load(itemUrl));
-
-            // 合并解析后的结果集作为该篇文章最终的输出结果
-            return Promise.resolve(Object.assign({}, single, other));
-        })
-    );
-module.exports = {
-    ProcessFeed,
-};

package/lib/routes/universities/cqwu/news.js

@@ -1,39 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url').resolve;
-
-const host = 'http://www.cqwu.edu.cn';
-const map = {
-    notify: '/channel_7721.html',
-    academiceve: '/channel_7722.html',
-};
-const titleMap = {
-    notify: '通知',
-    academiceve: '学术活动',
-};
-
-module.exports = async (ctx) => {
-    const type = ctx.params.type || 'academiceve';
-    const link = host + map[type];
-    const title = '重文理' + titleMap[type] + '公告';
-    const response = await got.get(link);
-    const $ = cheerio.load(response.data);
-    const list = $('ul[class="list-unstyled news-uls"]').find('li');
-    ctx.state.data = {
-        title,
-        link,
-        item:
-            list &&
-            list
-                .map((index, item) => {
-                    item = $(item);
-                    return {
-                        title: item.find('h4').text(),
-                        description: item.find('p').text(),
-                        pubDate: new Date(item.find('span[class="pull-right nes-date"]').text()).toUTCString(),
-                        link: url(host, item.find('a').attr('href')),
-                    };
-                })
-                .get(),
-    };
-};

package/lib/routes/universities/njupt/jwc.js

@@ -1,84 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-
-const host = 'http://jwc.njupt.edu.cn';
-
-const map = {
-    notice: '/1594',
-    news: '/1596',
-};
-
-module.exports = async (ctx) => {
-    const type = ctx.params.type || 'notice';
-    const link = host + map[type] + '/list.htm';
-    const response = await got({
-        method: 'get',
-        url: link,
-        headers: {
-            Referer: host,
-        },
-    });
-    const $ = cheerio.load(response.data);
-
-    const urlList = $('.content')
-        .find('a')
-        .slice(0, 10)
-        .map((i, e) => $(e).attr('href'))
-        .get();
-
-    const titleList = $('.content')
-        .find('a')
-        .slice(0, 10)
-        .map((i, e) => $(e).attr('title'))
-        .get();
-
-    const dateList = $('.content tr')
-        .find('div')
-        .slice(0, 10)
-        .map((i, e) => $(e).text().replace('发布时间：', ''))
-        .get();
-
-    const out = await Promise.all(
-        urlList.map(async (itemUrl, index) => {
-            itemUrl = url.resolve(host, itemUrl);
-            if (itemUrl.indexOf('.htm') !== -1) {
-                const cache = await ctx.cache.get(itemUrl);
-                if (cache) {
-                    return Promise.resolve(JSON.parse(cache));
-                }
-                const response = await got.get(itemUrl);
-                const $ = cheerio.load(response.data);
-                const single = {
-                    title: $('.Article_Title').text(),
-                    link: itemUrl,
-                    description: $('.wp_articlecontent')
-                        .html()
-                        .replace(/src="\//g, `src="${url.resolve(host, '.')}`)
-                        .replace(/href="\//g, `href="${url.resolve(host, '.')}`)
-                        .trim(),
-                    pubDate: new Date($('.Article_PublishDate').text().replace('发布时间：', '')).toUTCString(),
-                };
-                ctx.cache.set(itemUrl, JSON.stringify(single));
-                return Promise.resolve(single);
-            } else {
-                const single = {
-                    title: titleList[index],
-                    link: itemUrl,
-                    description: '该通知为文件，请点击原文链接↑下载',
-                    pubDate: new Date(dateList[index]).toUTCString(),
-                };
-                return Promise.resolve(single);
-            }
-        })
-    );
-    let info = '通知公告';
-    if (type === 'news') {
-        info = '教务快讯';
-    }
-    ctx.state.data = {
-        title: '南京邮电大学 -- ' + info,
-        link,
-        item: out,
-    };
-};

package/lib/routes/universities/nuaa/cs/index.js

@@ -1,84 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-const getCookie = require('../utils/pypasswaf');
-const host = 'http://cs.nuaa.edu.cn/';
-
-const map = new Map([
-    ['tzgg', { title: '南京航空航天大学计算机科学与技术学院 -- 通知公告', suffix: '1995/list.htm' }],
-    ['xwdt', { title: '南京航空航天大学计算机科学与技术学院 -- 新闻动态', suffix: '1997/list.htm' }],
-    ['kydt', { title: '南京航空航天大学计算机科学与技术学院 -- 科研动态', suffix: '1975/list.htm' }],
-    ['jxdt', { title: '南京航空航天大学计算机科学与技术学院 -- 教学动态', suffix: '1977/list.htm' }],
-    ['xsgz', { title: '南京航空航天大学计算机科学与技术学院 -- 学生工作', suffix: '1959/list.htm' }],
-    ['zsxx', { title: '南京航空航天大学计算机科学与技术学院 -- 招生信息', suffix: '1993/list.htm' }],
-    ['jyxx', { title: '南京航空航天大学计算机科学与技术学院 -- 就业信息', suffix: '1994/list.htm' }],
-]);
-
-module.exports = async (ctx) => {
-    const type = ctx.params.type || 'tzgg';
-    const suffix = map.get(type).suffix;
-
-    const link = url.resolve(host, suffix);
-    const cookie = await getCookie();
-    const gotConfig = {
-        headers: {
-            cookie,
-        },
-    };
-    const response = await got.get(link, gotConfig);
-    const $ = cheerio.load(response.data);
-
-    const list = $('#news_list > ul > li')
-        .slice(0, 10)
-        .map(function () {
-            const info = {
-                title: $(this).find('a').attr('title'),
-                link: $(this).find('a').attr('href'),
-                date: $(this).find('span').text(),
-            };
-            return info;
-        })
-        .get();
-
-    const out = await Promise.all(
-        list.map(async (info) => {
-            const title = info.title || 'tzgg';
-            const date = info.date;
-            const itemUrl = url.resolve(host, info.link);
-
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const arr = itemUrl.split('.');
-            const pageType = arr[arr.length - 1];
-
-            let description = itemUrl;
-            if (pageType === 'htm' || pageType === 'html') {
-                const response = await got.get(itemUrl, gotConfig);
-                const $ = cheerio.load(response.data);
-                description = $('.wp_articlecontent')
-                    .html()
-                    .replace(/src="\//g, `src="${url.resolve(host, '.')}`)
-                    .trim();
-            }
-
-            const single = {
-                title,
-                link: itemUrl,
-                description,
-                pubDate: new Date(date).toUTCString(),
-            };
-            ctx.cache.set(itemUrl, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
-    );
-
-    ctx.state.data = {
-        title: map.get(type).title,
-        link,
-        description: '南京航空航天大学计算机科学与技术学院RSS',
-        item: out,
-    };
-};

package/lib/routes/universities/nuaa/jwc/jwc.js

@@ -1,105 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-const host = 'http://aao.nuaa.edu.cn/';
-// https is not allowed in nuaa;interesting ha
-
-const map = {
-    default: 8230,
-    jxfw: 8230, // 教学服务
-    xspy: 8231, // 学生培养
-    jxjs: 8232, // 教学建设
-    jxzy: 8233, // 教学资源
-};
-
-async function load(link, cookie, ctx) {
-    const cache = await ctx.cache.get(link);
-    if (cache) {
-        return JSON.parse(cache);
-    }
-    const response = await got.get(link, {
-        headers: {
-            cookie,
-        },
-    });
-    const $ = cheerio.load(response.data);
-    const pubDate = new Date(
-        $('.release-time')
-            .text()
-            .slice(-10)
-            .match(/\d{4}-\d{2}-\d{2}/)
-    ).toUTCString();
-    const images = $('img');
-    for (let k = 0; k < images.length; k++) {
-        $(images[k]).replaceWith(`<img src="${url.resolve(host, $(images[k]).attr('src'))}" />`);
-    }
-    const description = $('.wp_articlecontent').html();
-    const result = { pubDate, description };
-    ctx.cache.set(link, JSON.stringify(result));
-
-    return result;
-}
-
-module.exports = async (ctx) => {
-    const browser = await require('@/utils/puppeteer')();
-    const type = ctx.params.type || 'default';
-    const listUrl = `${host}${map[type]}/list.htm`;
-
-    const page = await browser.newPage();
-    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36');
-    await page.evaluateOnNewDocument(() => {
-        // eslint-disable-next-line
-        Object.defineProperty(navigator, 'webdriver', { get: () => false });
-    });
-    // 南航的新waf现在采用cookie验证的方法防止爬虫，并且会对chrome headless进行检测
-
-    await page.goto(listUrl, {
-        waitUntil: 'networkidle0',
-    });
-
-    let msg;
-    try {
-        // data = await page.$('ul.right-ul')
-        msg = await page.$$eval('ul.right-ul > li', (es) =>
-            es.map((e) => {
-                const html = e.innerHTML;
-
-                let [, title] = html.match(/<a.+?>(.+)<\/a>/);
-                if (title.match(/font/)) {
-                    [, title] = title.match(/>(.+)</);
-                }
-                const [, href] = html.match(/href="(.+?)"/);
-                // const [, time] = html.match(/-time">(.+?)</);
-                return {
-                    title,
-                    link: href,
-                    guid: href,
-                };
-            })
-        );
-    } catch (e) {
-        // debugger
-    }
-
-    let cookie = await page.cookies();
-    cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; ');
-    browser.close();
-    msg = await Promise.all(
-        msg.map(async (e) => {
-            const link = url.resolve(host, e.link);
-            return {
-                ...e,
-                link,
-                guid: link,
-                ...(await load(link, cookie, ctx)),
-            };
-        })
-    );
-
-    ctx.state.data = {
-        title: '南航教务',
-        link: host,
-        description: '南航教务RSS',
-        item: msg,
-    };
-};

package/lib/routes/universities/nuaa/utils/pypasswaf.js

@@ -1,24 +0,0 @@
-const host = 'http://aao.nuaa.edu.cn/';
-
-/**
- * async function 获取cookie
- * @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.header.cookie`即可
- */
-module.exports = async function getCookie() {
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36');
-    await page.evaluateOnNewDocument(() => {
-        // eslint-disable-next-line
-        Object.defineProperty(navigator, 'webdriver', { get: () => false });
-    });
-
-    await page.goto(host, {
-        waitUntil: 'networkidle0',
-    });
-
-    let cookie = await page.cookies();
-    cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; ');
-    await browser.close();
-    return cookie;
-};

package/lib/routes/universities/nuaa/yjsy/yjsy.js

@@ -1,61 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-const host = 'http://www.graduate.nuaa.edu.cn/';
-const formatPubDate = require('@/utils/date.js');
-
-const map = {
-    latest: '2146/list.htm',
-    yyxw: 'yyxw/list.htm',
-    sjwj: 'sjwj/list.htm',
-    glwj: '2017/list.htm',
-    xxfw: '2147/list.htm',
-};
-
-async function load(link, ctx) {
-    const cache = await ctx.cache.get(link);
-    if (cache) {
-        return cache;
-    }
-    const response = await got.get(host + link);
-    const $ = cheerio.load(response.data);
-    const images = $('img');
-    for (let k = 0; k < images.length; k++) {
-        $(images[k]).replaceWith(`<img src="${url.resolve(host, $(images[k]).attr('src'))}" />`);
-    }
-    const description = $('.wp_articlecontent').html();
-    ctx.cache.set(link, description);
-    return { description };
-}
-
-module.exports = async (ctx) => {
-    const type = ctx.params.type || 'latest';
-    const response = await got({
-        method: 'get',
-        url: host + map[type],
-    });
-    const $ = cheerio.load(response.data);
-    const list = $('#news_list ul.news_ul > li').slice(0, 10).get();
-    const process = await Promise.all(
-        list.map(async (item) => {
-            const a = $(item).find('a');
-            const t = $(item).find('span');
-            const itemUrl = a.attr('href');
-            const single = {
-                title: a.text(),
-                link: url.resolve(host, itemUrl),
-                guid: url.resolve(host, itemUrl),
-                pubDate: formatPubDate(t.text()),
-            };
-            const other = await load(itemUrl, ctx);
-            return Promise.resolve(Object.assign({}, single, other));
-        })
-    );
-
-    ctx.state.data = {
-        title: '南航研究生院',
-        link: host,
-        description: '南航研究生院RSS',
-        item: process,
-    };
-};

package/lib/routes/universities/sysu/sdcs.js

@@ -1,103 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: 'http://sdcs.sysu.edu.cn/',
-        headers: {
-            Referer: `http://sdcs.sysu.edu.cn/`,
-        },
-    });
-    const $ = cheerio.load(response.data);
-
-    // 首页有多个板块，每个板块的css选择器不同，而且每个板块的信息分类也不一样
-    const block_index = [
-        {
-            index: 1,
-            description_header: '学院新闻',
-        },
-        {
-            index: 2,
-            description_header: '学院通知',
-        },
-        {
-            index: 3,
-            description_header: '人才招聘',
-        },
-        {
-            index: 4,
-            description_header: '学术活动',
-        },
-        {
-            index: 5,
-            description_header: '学工通知',
-        },
-        {
-            index: 6,
-            description_header: '学生活动',
-        },
-        {
-            index: 7,
-            description_header: '教务通知',
-        },
-        {
-            index: 8,
-            description_header: '科研通知',
-        },
-        {
-            index: 9,
-            description_header: '人事通知',
-        },
-        {
-            index: 10,
-            description_header: '党群工作',
-        },
-        {
-            index: 11,
-            description_header: '校友工作',
-        },
-        {
-            index: 12,
-            description_header: '社会工作',
-        },
-    ];
-
-    function getDetail(item, description_header) {
-        return {
-            title: description_header + ': ' + item.attribs.title,
-            description: description_header + ': ' + item.attribs.title,
-            link: item.attribs.href,
-            category: description_header,
-        };
-    }
-
-    const item_data = [];
-    for (let i = 0; i < block_index.length; i++) {
-        const block_news = $('#block-views-homepage-block-' + block_index[i].index + '> div > div.view-content > div > ul > li > a');
-        for (let j = 0; j < block_news.length; j++) {
-            item_data.push(getDetail(block_news[j], block_index[i].description_header));
-        }
-    }
-
-    function compareLink(a, b) {
-        let a_str = a.link;
-        a_str = a_str.substr(a_str.length - 4, 4);
-        const a_int = parseInt(a_str);
-        let b_str = b.link;
-        b_str = b_str.substr(b_str.length - 4, 4);
-        const b_int = parseInt(b_str);
-        return b_int - a_int;
-    }
-    // 使得新的通知排在前面，假设通知的发布和链接地址是相关的，而且链接地址都是"/content/4961"这样，只有四位数的。
-    item_data.sort(compareLink);
-    // console.log(item_data);
-
-    ctx.state.data = {
-        title: `中山大学 - 数据科学与计算机学院`,
-        link: `http://sdcs.sysu.edu.cn`,
-        description: `中山大学 - 数据科学与计算机学院`,
-        language: `zh-cn`,
-        item: item_data,
-    };
-};

package/lib/routes/verfghbw/press.js

@@ -1,52 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const keyword = ctx.params.keyword;
-
-    let request = {
-        url: 'https://verfgh.baden-wuerttemberg.de/de/presse-und-service/pressemitteilungen/',
-        headers: {
-            Referer: 'https://verfgh.baden-wuerttemberg.de/de/presse-und-service/pressemitteilungen/',
-        },
-    };
-
-    if (keyword) {
-        request = {
-            method: 'post',
-            form: {
-                'tx_bwlistheader_list[search][keywords]': keyword,
-            },
-            ...request,
-        };
-    } else {
-        request = {
-            method: 'get',
-            ...request,
-        };
-    }
-
-    const response = await got(request);
-
-    const data = response.data;
-    const $ = cheerio.load(data);
-
-    const list = $('.pressListItem')
-        .map((_, item) => {
-            item = $(item);
-
-            return {
-                title: item.find('.pressListItemTeaser > h3').text().trim(),
-                link: new URL(item.find('.link-download').attr('href'), 'https://verfgh.baden-wuerttemberg.de/'),
-                pubDate: new Date(item.find('.pressListItemDate > span').text()).toUTCString(),
-            };
-        })
-        .get();
-
-    ctx.state.data = {
-        title: 'Verfassungsgerichtshof Baden-Württemberg - Pressemitteilungen',
-        link: request.url,
-        description: 'Pressemitteilungen des Verfassungsgerichtshof für das Land Baden-Württemberg',
-        item: list,
-    };
-};

package/lib/routes/wallstreetcn/live.js

@@ -1,44 +0,0 @@
-const got = require('@/utils/got');
-
-const config = {
-    global: '要闻',
-    'a-stock': 'A股',
-    'us-stock': '美股',
-    'hk-stock': '港股',
-    forex: '外汇',
-    commodity: '商品',
-    financing: '理财',
-};
-
-module.exports = async (ctx) => {
-    ctx.params.channel = ctx.params.channel || 'global';
-
-    const currentUrl = `https://wallstreetcn.com/live/${ctx.params.channel}`;
-    const rootUrl = `https://api.wallstcn.com/apiv1/content/lives?channel=${ctx.params.channel}-channel`;
-    const response = await got({
-        method: 'get',
-        url: rootUrl,
-    });
-
-    const items = response.data.data.items.map((item) => {
-        let title;
-        const match = item.content_text.match(/【(.*)】/);
-        if (match !== null) {
-            title = match[1];
-        } else {
-            title = item.content_text;
-        }
-        return {
-            title,
-            link: `https://wallstreetcn.com/livenews/${item.id}`,
-            pubDate: new Date(item.display_time * 1000).toUTCString(),
-            description: item.content,
-        };
-    });
-
-    ctx.state.data = {
-        title: `${config[ctx.params.channel]} - 实时快讯 - 华尔街见闻`,
-        link: currentUrl,
-        item: items,
-    };
-};