rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/v2/bjx/huanbao.js

@@ -0,0 +1,78 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const asyncPool = require('tiny-async-pool');
+
+const asyncPoolAll = async (...args) => {
+    const results = [];
+    for await (const result of asyncPool(...args)) {
+        results.push(result);
+    }
+    return results;
+};
+
+module.exports = async (ctx) => {
+    const listURL = 'https://huanbao.bjx.com.cn/yw/';
+    const response = await got(listURL);
+
+    const $ = cheerio.load(response.data);
+    let items = $('.cc-layout-3 .cc-list-content li')
+        .toArray()
+        .map((e) => {
+            e = $(e);
+            return {
+                title: e.find('a').attr('title'),
+                link: e.find('a').attr('href'),
+                pubDate: parseDate(e.find('span').text()),
+            };
+        });
+
+    items = await asyncPoolAll(
+        // 服务器禁止单个IP大并发访问，只能少返回几条
+        3,
+        items,
+        (items) => fetchPage(ctx, items.link)
+    );
+
+    ctx.state.data = {
+        title: '北极星环保 - 环保行业垂直门户网站',
+        link: listURL,
+        item: items,
+    };
+};
+
+const fetchPage = (ctx, link) =>
+    ctx.cache.tryGet(link, async () => {
+        // 可能一篇文章过长会分成多页
+        const pages = [];
+
+        const result = await got(link);
+        const $page = cheerio.load(result.data);
+        pages.push($page);
+
+        // 如果是有分页链接，则使用顺序加载以保证顺序
+        const pagelinks = $page('#article_cont .cc-paging a');
+
+        if (pagelinks.length > 0) {
+            for (let i = 0; i < pagelinks.length; i++) {
+                const $a = $page(pagelinks[i]);
+                if (!/^\d+$/.test($a.text().trim())) {
+                    continue;
+                }
+                const sublink = new URL($a.attr('href'), link).href;
+                /* eslint-disable no-await-in-loop */
+                const result = await got(sublink);
+                pages.push(cheerio.load(result.data));
+            }
+        }
+
+        const item = {
+            title: $page('title').text(),
+            description: pages.reduce((desc, $p) => desc + $p('.cc-article').html(), ''),
+            pubDate: timezone(parseDate($page('.cc-headline .box p span').eq(0).text()), +8),
+            link,
+            author: $page('.cc-headline .box p span').eq(1).text(),
+        };
+        return item;
+    });

package/lib/v2/bjx/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/gf/:type': ['Sxuet'],
+    '/huanbao': ['zsimple'],
+};

package/lib/v2/bjx/radar.js

@@ -0,0 +1,21 @@
+module.exports = {
+    'bjx.com.cn': {
+        _name: '北极星电力网',
+        guangfu: [
+            {
+                title: '光伏 - 分类',
+                docs: 'https://docs.rsshub.app/traditional-media.html#bei-ji-xing-dian-li-wang',
+                source: ['/:type', '/'],
+                target: '/bjx/gf/:type?',
+            },
+        ],
+        huanbao: [
+            {
+                title: '环保要闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#bei-ji-xing-dian-li-wang',
+                source: ['/yw', '/'],
+                target: '/bjx/huanbao',
+            },
+        ],
+    },
+};

package/lib/v2/bjx/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/gf/:type', require('./types'));
+    router.get('/huanbao', require('./huanbao'));
+};

package/lib/v2/bjx/types.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type;
+    const response = await got({
+        method: 'get',
+        url: `https://guangfu.bjx.com.cn/${type}/`,
+    });
+    const data = response.data;
+    const $ = cheerio.load(data);
+    const typeName = $('div.box2 em:last').text();
+    const list = $('div.cc-list-content ul li');
+    ctx.state.data = {
+        title: `北极星太阳能光大网${typeName}`,
+        description: $('meta[name="Description"]').attr('content'),
+        link: `https://guangfu.bjx.com.cn/${type}/`,
+        item:
+            list &&
+            list
+                .map((index, item) => {
+                    item = $(item);
+                    return {
+                        title: item.find('a').attr('title'),
+                        description: item.html(),
+                        link: item.find('a').attr('href'),
+                        pubDate: parseDate(item.find('span').text()),
+                    };
+                })
+                .get(),
+    };
+};

package/lib/v2/bookfere/category.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const url = 'https://bookfere.com/category/' + ctx.params.category;
+    const response = await got({
+        method: 'get',
+        url,
+    });
+
+    const data = response.data;
+
+    const $ = cheerio.load(data);
+    const list = $('main div div section');
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        item:
+            list &&
+            list
+                .map((index, item) => {
+                    item = $(item);
+                    const date = item.find('time').attr('datetime');
+                    const pubDate = parseDate(date);
+                    return {
+                        title: item.find('h2 a').text(),
+                        link: item.find('h2 a').attr('href'),
+                        pubDate,
+                        description: item.find('p').text(),
+                    };
+                })
+                .get(),
+    };
+};

package/lib/v2/bookfere/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category': ['OdinZhang'],
+};

package/lib/v2/bookfere/radar.js

@@ -0,0 +1,11 @@
+module.exports = {
+    'bookfere.com': {
+        _name: '书伴',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/reading.html#shu-ban',
+            },
+        ],
+    },
+};

package/lib/v2/bookfere/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:category', require('./category'));
+};

package/lib/v2/brooklynmuseum/exhibitions.js

@@ -0,0 +1,27 @@
+const buildData = require('@/utils/common-config');
+
+module.exports = async (ctx) => {
+    let link;
+    const state = ctx.params.state;
+
+    switch (state) {
+        case undefined:
+        case 'current':
+            link = 'https://www.brooklynmuseum.org/exhibitions/';
+            break;
+        default:
+            link = `https://www.brooklynmuseum.org/exhibitions/${state}`;
+    }
+
+    ctx.state.data = await buildData({
+        link,
+        url: link,
+        title: 'Brooklyn Museum - Exhibitions',
+        item: {
+            item: '.exhibitions .image-card',
+            title: `$('h2 > a, h3 > a').text()`,
+            link: `$('h2 > a, h3 > a').attr('href')`,
+            description: `$('h6').text()`,
+        },
+    });
+};

package/lib/v2/brooklynmuseum/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/exhibitions': ['chazeon'],
+};

package/lib/v2/brooklynmuseum/radar.js

@@ -0,0 +1,11 @@
+module.exports = {
+    'brooklynmuseum.org': {
+        _name: 'Brooklyn Museum',
+        www: [
+            {
+                title: 'Exhibitions',
+                docs: 'https://docs.rsshub.app/en/travel.html#brooklyn-museum',
+            },
+        ],
+    },
+};

package/lib/v2/brooklynmuseum/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/exhibitions/:state?', require('./exhibitions'));
+};

package/lib/v2/c114/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/roll': ['nczitzk'],
+};

package/lib/v2/c114/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'c114.com.cn': {
+        _name: 'C114通信网',
+        '.': [
+            {
+                title: '滚动新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#c114-tong-xin-wang-gun-dong-xin-wen',
+                source: ['/news/roll.asp', '/'],
+                target: '/c114/roll',
+            },
+        ],
+    },
+};

package/lib/v2/c114/roll.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const iconv = require('iconv-lite');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.c114.com.cn';
+    const currentUrl = `${rootUrl}/news/roll.asp`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+        responseType: 'buffer',
+    });
+
+    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+
+    let items = $('.new_list_c h6 a')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                    responseType: 'buffer',
+                });
+
+                const content = cheerio.load(iconv.decode(detailResponse.data, 'gbk'));
+
+                item.description = content('.text').html();
+                item.author = content('.author').first().text().replace('C114通信网  ', '');
+                item.pubDate = timezone(parseDate(content('.r_time').text()), +8);
+                item.category = content('meta[name="keywords"]').attr('content').split(',');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/c114/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/roll', require('./roll'));
+};

package/lib/v2/caareviews/book.js

@@ -0,0 +1,14 @@
+const { rootUrl, getList, getItems } = require('./utils');
+
+module.exports = async (ctx) => {
+    const url = `${rootUrl}/reviews/book`;
+
+    const list = await getList(url);
+    const items = await getItems(ctx, list);
+
+    ctx.state.data = {
+        title: 'Book Reviews',
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/caareviews/essay.js

@@ -0,0 +1,14 @@
+const { rootUrl, getList, getItems } = require('./utils');
+
+module.exports = async (ctx) => {
+    const url = `${rootUrl}/reviews/essay`;
+
+    const list = await getList(url);
+    const items = await getItems(ctx, list);
+
+    ctx.state.data = {
+        title: 'Essays',
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/caareviews/exhibition.js

@@ -0,0 +1,14 @@
+const { rootUrl, getList, getItems } = require('./utils');
+
+module.exports = async (ctx) => {
+    const url = `${rootUrl}/reviews/exhibition`;
+
+    const list = await getList(url);
+    const items = await getItems(ctx, list);
+
+    ctx.state.data = {
+        title: 'Exhibition Reviews',
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/caareviews/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/book': ['Fatpandac'],
+    '/essay': ['Fatpandac'],
+    '/exhibition': ['Fatpandac'],
+};

package/lib/v2/caareviews/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'caareviews.org': {
+        _name: 'caa.reviews',
+        '.': [
+            {
+                title: 'Book Reviews',
+                docs: 'https://docs.rsshub.app/journal.html#caa-reviews',
+                source: ['/reviews/book'],
+                target: '/caareviews/book',
+            },
+            {
+                title: 'Exhibition Reviews',
+                docs: 'https://docs.rsshub.app/journal.html#caa-reviews',
+                source: ['/reviews/exhibition'],
+                target: '/caareviews/exhibition',
+            },
+            {
+                title: 'Essays',
+                docs: 'https://docs.rsshub.app/journal.html#caa-reviews',
+                source: ['/reviews/essay'],
+                target: '/caareviews/essay',
+            },
+        ],
+    },
+};

package/lib/v2/caareviews/router.js

@@ -0,0 +1,5 @@
+module.exports = function (router) {
+    router.get('/book', require('./book'));
+    router.get('/essay', require('./essay'));
+    router.get('/exhibition', require('./exhibition'));
+};

package/lib/v2/caareviews/templates/utils.art

@@ -0,0 +1,2 @@
+<img src="{{ coverUrl }}">
+{{@ content}}

package/lib/v2/caareviews/utils.js

@@ -0,0 +1,49 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const rootUrl = 'http://www.caareviews.org';
+
+const getList = async (url) => {
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const list = $('#infinite-content > div')
+        .map((_index, item) => ({
+            title: $(item).find('div.title').text().trim(),
+            link: new URL($(item).find('div.title > em > a').attr('href'), rootUrl).href,
+            author: $(item).find('div.contributors').text().trim(),
+        }))
+        .get();
+
+    return list;
+};
+
+const getItems = async (ctx, list) =>
+    await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const $ = cheerio.load(detailResponse.data);
+
+                const coverUrl = new URL($('div.cover > a').attr('href'), rootUrl).href;
+                const content = $('div.content.full-review').html();
+                item.description = art(path.join(__dirname, 'templates/utils.art'), {
+                    coverUrl,
+                    content,
+                });
+                $('div.review_heading').remove();
+                item.pubDate = parseDate($('div.header-text > div.clearfix').text());
+                item.doi = $('div.crossref > a').attr('href').replace('http://dx.doi.org/', '');
+
+                return item;
+            })
+        )
+    );
+
+module.exports = {
+    rootUrl,
+    getList,
+    getItems,
+};

package/lib/v2/cankaoxiaoxi/index.js

@@ -0,0 +1,81 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const cheerio = require('cheerio');
+
+const nodes = {
+    china_news: {
+        title: '中国',
+        url: 'https://china.cankaoxiaoxi.com/',
+    },
+    world_news: {
+        title: '国际',
+        url: 'https://world.cankaoxiaoxi.com/',
+    },
+    military_news: {
+        title: '军事',
+        url: 'https://mil.cankaoxiaoxi.com/',
+    },
+    taiwan_news: {
+        title: '台海',
+        url: 'https://tw.cankaoxiaoxi.com/',
+    },
+    finance_news: {
+        title: '财经',
+        url: 'https://finance.cankaoxiaoxi.com/',
+    },
+    technology_news: {
+        title: '科技',
+        url: 'https://science.cankaoxiaoxi.com/',
+    },
+    culture_news: {
+        title: '文化',
+        url: 'https://culture.cankaoxiaoxi.com/',
+    },
+};
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+
+    const currentUrl = nodes[category].url;
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('a', '.toutiao')
+        .filter(function () {
+            return $(this).text() !== '';
+        })
+        .map((_, item) => ({
+            link: $(item).attr('href'),
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(detailResponse.data);
+                item.title = content('h1.articleHead').text();
+                item.pubDate = timezone(parseDate(content('span#pubtime_baidu').text()), +8);
+                item.author = content('span#source_baidu').text();
+                item.description = content('div.articleText').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${nodes[category].title} 参考消息`,
+        link: currentUrl,
+        description: '参考消息',
+        language: 'zh-cn',
+        item: items,
+    };
+};

package/lib/v2/cankaoxiaoxi/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:category': ['yuxinliu-alex'],
+};

package/lib/v2/cankaoxiaoxi/radar.js

@@ -0,0 +1,61 @@
+module.exports = {
+    'cankaoxiaoxi.com': {
+        _name: '参考消息',
+        china: [
+            {
+                title: '中国新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/china_news',
+            },
+        ],
+        culture: [
+            {
+                title: '文化新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/culture_news',
+            },
+        ],
+        finance: [
+            {
+                title: '财经新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/finance_news',
+            },
+        ],
+        mil: [
+            {
+                title: '军事新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/military_news',
+            },
+        ],
+        science: [
+            {
+                title: '科技新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/technology_news',
+            },
+        ],
+        tw: [
+            {
+                title: '台海新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/taiwan_news',
+            },
+        ],
+        world: [
+            {
+                title: '国际新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/world_news',
+            },
+        ],
+    },
+};

package/lib/v2/cankaoxiaoxi/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/news/:category', require('./index'));
+};

package/lib/v2/cbirc/index.js

@@ -86,7 +86,7 @@ module.exports = async (ctx) => {
         const resp = await got({
             method: 'get',
             url: cat.link,
-            header: {
+            headers: {
                 Referer: `http://www.cbirc.gov.cn`,
             },
         });