rsshub 1.0.0-master.f9a482d → 1.0.0-master.fa27a51

package/lib/v2/bjfu/it/index.js

@@ -1,7 +1,7 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const util = require('./utils');
-const iconv = require('iconv-lite'); // 转码
+const iconv = require('iconv-lite');
 
 module.exports = async (ctx) => {
     const type = ctx.params.type;
@@ -27,16 +27,18 @@ module.exports = async (ctx) => {
 
     const response = await got({
         method: 'get',
-        responseType: 'buffer', // 转码
+        responseType: 'buffer',
         url: base,
     });
 
-    const data = iconv.decode(response.data, 'gb2312'); // 转码
-    const $ = cheerio.load(data);
-
-    // const list = $('div[item-content]').slice(0, 10).get();
+    const data = response.data;
+    let $ = cheerio.load(iconv.decode(data, 'utf-8'));
+    const charset = $('meta[charset]').attr('charset');
+    if (charset?.toLowerCase() !== 'utf-8') {
+        $ = cheerio.load(iconv.decode(data, charset ?? 'utf-8'));
+    }
 
-    const list = $('.item-content').get();
+    const list = $('.item-content').toArray();
 
     const result = await util.ProcessFeed(base, list, ctx.cache); // 感谢@hoilc指导
 

package/lib/v2/bjfu/it/utils.js

@@ -1,62 +1,71 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const iconv = require('iconv-lite'); // 转码
+const iconv = require('iconv-lite');
 const { parseDate } = require('@/utils/parse-date');
 const timezone = require('@/utils/timezone');
 
 // 完整文章页
 async function load(link) {
-    const response = await got.get(link, {
-        responseType: 'buffer',
-    });
+    let response;
+    try {
+        response = await got.get(link, {
+            responseType: 'buffer',
+        });
+    } catch (e) {
+        return { description: '' };
+    }
 
-    const data = iconv.decode(response.data, 'gb2312'); // 转码
+    const data = response.data; // 不用转码
 
     // 加载文章内容
-    const $ = cheerio.load(data);
-
-    // 解析日期
-    const pubDate = timezone(
-        parseDate(
-            $('.template-head-info')
-                .text()
-                .match(/\d{4}-\d{2}-\d{2}/)
-        ),
-        +8
-    );
+    let $ = cheerio.load(iconv.decode(data, 'utf-8'));
+    const charset = $('meta[charset]').attr('charset');
+    if (charset?.toLowerCase() !== 'utf-8') {
+        $ = cheerio.load(iconv.decode(data, charset ?? 'utf-8'));
+    }
 
     // 提取内容
-    const description = $('.template-body').html();
+    const description = ($('.template-body').length ? $('.template-body').html() : '') + ($('.template-tail').length ? $('.template-tail').html() : '');
 
     // 返回解析的结果
-    return { description, pubDate };
+    return { description };
 }
 
 const ProcessFeed = (base, list, caches) =>
     // 使用 Promise.all() 进行 async 并发
     Promise.all(
         // 遍历每一篇文章
-        list.map(async (item) => {
+        list.map((item) => {
             const $ = cheerio.load(item);
 
             const $title = $('a');
             // 还原相对链接为绝对链接
             const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
 
-            // 列表上提取到的信息
-            const single = {
-                title: $title.text(),
-                link: itemUrl,
-                author: '北林信息',
-                guid: itemUrl,
-            };
+            // 解析日期
+            const pubDate = timezone(
+                parseDate(
+                    $('span')
+                        .text()
+                        .match(/\d{4}-\d{2}-\d{2}/)
+                ),
+                +8
+            );
 
             // 使用tryGet方法从缓存获取内容。
             // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
-            const other = await caches.tryGet(itemUrl, () => load(itemUrl));
+            return caches.tryGet(itemUrl, async () => {
+                const { description } = await load(itemUrl);
 
-            // 合并解析后的结果集作为该篇文章最终的输出结果
-            return { ...single, ...other };
+                // 列表上提取到的信息
+                return {
+                    title: $title.text(),
+                    link: itemUrl,
+                    author: '北林信息',
+                    description,
+                    pubDate,
+                };
+            });
         })
     );
 module.exports = {

package/lib/v2/bjfu/jwc/index.js

@@ -1,16 +1,11 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const util = require('./utils');
-const iconv = require('iconv-lite'); // 转码
 
 module.exports = async (ctx) => {
     const type = ctx.params.type;
     let title, path;
     switch (type) {
-        case 'jwkx':
-            title = '教务快讯';
-            path = 'jwkx/';
-            break;
         case 'jgdt':
             title = '教改动态';
             path = 'jgdt/';
@@ -27,6 +22,7 @@ module.exports = async (ctx) => {
             title = '图片新闻';
             path = 'tpxw/';
             break;
+        case 'jwkx':
         default:
             title = '教务快讯';
             path = 'jwkx/';
@@ -35,14 +31,13 @@ module.exports = async (ctx) => {
 
     const response = await got({
         method: 'get',
-        responseType: 'buffer', // 转码
         url: base,
     });
 
-    const data = iconv.decode(response.data, 'gb2312'); // 转码
+    const data = response.data; // 不用转码
     const $ = cheerio.load(data);
 
-    const list = $('.list_c li').slice(0, 10).get();
+    const list = $('.list_c li').slice(0, 15).toArray();
 
     const result = await util.ProcessFeed(base, list, ctx.cache); // 感谢@hoilc指导
 

package/lib/v2/bjfu/jwc/utils.js

@@ -1,61 +1,58 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const iconv = require('iconv-lite'); // 转码
 const { parseDate } = require('@/utils/parse-date');
 const timezone = require('@/utils/timezone');
 
 // 完整文章页
 async function load(link) {
-    const response = await got.get(link, {
-        responseType: 'buffer',
-    });
+    const response = await got.get(link);
 
-    const data = iconv.decode(response.data, 'gb2312'); // 转码
+    const data = response.data; // 不用转码
 
     // 加载文章内容
     const $ = cheerio.load(data);
 
-    // 解析日期
-    const pubDate = timezone(
-        parseDate(
-            $('div #con_djl')
-                .text()
-                .match(/\d{4}-\d{2}-\d{2}/)
-        ),
-        +8
-    );
-
     // 提取内容
-    const description = $('#con_c').html();
+    const description = ($('#con_c').length ? $('#con_c').html() : '') + ($('#con_fujian').length ? $('#con_fujian').html() : '');
 
     // 返回解析的结果
-    return { description, pubDate };
+    return { description };
 }
 
 const ProcessFeed = (base, list, caches) =>
     Promise.all(
         // 遍历每一篇文章
-        list.map(async (item) => {
+        list.map((item) => {
             const $ = cheerio.load(item);
 
             const $title = $('a');
             // 还原相对链接为绝对链接
             const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
 
-            // 列表上提取到的信息
-            const single = {
-                title: $title.text(),
-                link: itemUrl,
-                author: '北林教务处',
-                guid: itemUrl,
-            };
+            // 解析日期
+            const pubDate = timezone(
+                parseDate(
+                    $('.datetime')
+                        .text()
+                        .match(/\d{4}-\d{2}-\d{2}/)
+                ),
+                +8
+            );
 
             // 使用tryGet方法从缓存获取内容。
             // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
-            const other = await caches.tryGet(itemUrl, () => load(itemUrl));
-
-            // 合并解析后的结果集作为该篇文章最终的输出结果
-            return { ...single, ...other };
+            return caches.tryGet(itemUrl, async () => {
+                const { description } = await load(itemUrl);
+
+                // 列表上提取到的信息
+                return {
+                    title: $title.text(),
+                    link: itemUrl,
+                    author: '北林教务处',
+                    description,
+                    pubDate,
+                };
+            });
         })
     );
 module.exports = {

package/lib/v2/bjfu/kjc.js

@@ -1,60 +1,47 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const iconv = require('iconv-lite');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
     const url = 'http://kyc.bjfu.edu.cn/tztg/index.html';
-    const response = await got.get(url, {
-        responseType: 'buffer',
-    });
-    const data = iconv.decode(response.data, 'gb2312');
+    const response = await got.get(url);
+    const data = response.data;
     const $ = cheerio.load(data);
     const list = $('.ll_con_r_b li')
-        .slice(0, 10)
-        .map((i, e) => {
+        .slice(0, 15)
+        .toArray()
+        .map((e) => {
             const element = $(e);
             const title = element.find('.ll_con_r_b_title a').text();
             const link = element.find('a').attr('href');
-            const date = new Date(
-                element
-                    .find('.ll_con_r_b_time')
-                    .text()
-                    .match(/\d{4}-\d{2}-\d{2}/)
-            );
-            const timeZone = 8;
-            const serverOffset = date.getTimezoneOffset() / 60;
-            const pubDate = new Date(date.getTime() - 60 * 60 * 1000 * (timeZone + serverOffset)).toUTCString();
+            const date = element
+                .find('.ll_con_r_b_time')
+                .text()
+                .match(/\d{4}-\d{2}-\d{2}/);
+            const pubDate = timezone(parseDate(date), 8);
 
             return {
                 title,
-                description: '',
                 link: 'http://kyc.bjfu.edu.cn/tztg/' + link,
                 author: '北京林业大学科技处通知公告',
                 pubDate,
             };
-        })
-        .get();
+        });
 
     const result = await Promise.all(
-        list.map(async (item) => {
-            const link = item.link;
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const itemReponse = await got.get(link, {
-                responseType: 'buffer',
-            });
-            const data = iconv.decode(itemReponse.data, 'gb2312');
-            const itemElement = cheerio.load(data);
-
-            item.description = itemElement('#a_con_l_con').html();
-
-            ctx.cache.set(link, JSON.stringify(item));
-            return item;
-        })
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const itemReponse = await got.get(item.link);
+                const data = itemReponse.data;
+                const itemElement = cheerio.load(data);
+
+                item.description = itemElement('#a_con_l_con').html();
+                item.title = item.title.includes('...') ? itemElement('#a_con_l_title').text() : item.title;
+
+                return item;
+            })
+        )
     );
 
     ctx.state.data = {

package/lib/v2/bjfu/news/index.js

@@ -1,16 +1,12 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const util = require('./utils');
-const iconv = require('iconv-lite'); // 转码
+const iconv = require('iconv-lite');
 
 module.exports = async (ctx) => {
     const type = ctx.params.type;
     let title, path;
     switch (type) {
-        case 'lsyw':
-            title = '绿色要闻';
-            path = 'lsyw/';
-            break;
         case 'xydt':
             title = '校园动态';
             path = 'lsxy/';
@@ -27,6 +23,7 @@ module.exports = async (ctx) => {
             title = '一周排行';
             path = 'yzph/';
             break;
+        case 'lsyw':
         default:
             title = '绿色要闻';
             path = 'lsyw/';
@@ -35,14 +32,20 @@ module.exports = async (ctx) => {
 
     const response = await got({
         method: 'get',
-        responseType: 'buffer', // 转码
+        responseType: 'buffer',
         url: base,
     });
 
-    const data = iconv.decode(response.data, 'gb2312'); // 转码
-    const $ = cheerio.load(data);
+    const data = response.data;
+    let $ = cheerio.load(iconv.decode(data, 'utf-8'));
+    const charset = $('meta[http-equiv="Content-Type"]')
+        .attr('content')
+        .match(/charset=(.*)/)?.[1];
+    if (charset?.toLowerCase() !== 'utf-8') {
+        $ = cheerio.load(iconv.decode(data, charset ?? 'utf-8'));
+    }
 
-    const list = $('.news_ul li').slice(0, 10).get();
+    const list = $('.news_ul li').slice(0, 12).toArray();
 
     const result = await util.ProcessFeed(base, list, ctx.cache); // 感谢@hoilc指导
 

package/lib/v2/bjfu/news/utils.js

@@ -1,16 +1,13 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const iconv = require('iconv-lite'); // 转码
 const { parseDate } = require('@/utils/parse-date');
 const timezone = require('@/utils/timezone');
 
 // 完整文章页
 async function load(link) {
-    const response = await got.get(link, {
-        responseType: 'buffer',
-    });
+    const response = await got.get(link);
 
-    const data = iconv.decode(response.data, 'gb2312'); // 转码
+    const data = response.data; // 不用转码
 
     // 加载文章内容
     const $ = cheerio.load(data);
@@ -27,35 +24,36 @@ async function load(link) {
 
     // 提取内容
     const description = $('.article_con').html();
+    const title = $('h2').text();
 
     // 返回解析的结果
-    return { description, pubDate };
+    return { description, pubDate, title };
 }
 
 const ProcessFeed = (base, list, caches) =>
     Promise.all(
         // 遍历每一篇文章
-        list.map(async (item) => {
+        list.map((item) => {
             const $ = cheerio.load(item);
 
             const $title = $('a');
             // 还原相对链接为绝对链接
             const itemUrl = new URL($title.attr('href'), base).href; // 感谢@hoilc指导
 
-            // 列表上提取到的信息
-            const single = {
-                title: $title.text(),
-                link: itemUrl,
-                author: '绿色新闻网',
-                guid: itemUrl,
-            };
-
             // 使用tryGet方法从缓存获取内容。
             // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
-            const other = await caches.tryGet(itemUrl, () => load(itemUrl));
-
-            // 合并解析后的结果集作为该篇文章最终的输出结果
-            return { ...single, ...other };
+            return caches.tryGet(itemUrl, async () => {
+                const { description, pubDate, title } = await load(itemUrl);
+
+                // 列表上提取到的信息
+                return {
+                    title: $title.text().includes('...') ? title : $title.text(),
+                    link: itemUrl,
+                    author: '绿色新闻网',
+                    description,
+                    pubDate,
+                };
+            });
         })
     );
 module.exports = {

package/lib/v2/bjfu/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         graduate: [
             {
                 title: '研究生院培养动态',
-                docs: 'https://docs.rsshub.app/university.html#bei-jing-lin-ye-da-xue',
+                docs: 'https://docs.rsshub.app/routes/university#bei-jing-lin-ye-da-xue',
                 source: '/',
                 target: '/bjfu/grs',
             },
@@ -12,7 +12,7 @@ module.exports = {
         it: [
             {
                 title: '信息学院通知',
-                docs: 'https://docs.rsshub.app/university.html#bei-jing-lin-ye-da-xue',
+                docs: 'https://docs.rsshub.app/routes/university#bei-jing-lin-ye-da-xue',
                 source: '/:type/index.html',
                 target: '/bjfu/it/:type',
             },
@@ -20,7 +20,7 @@ module.exports = {
         jwc: [
             {
                 title: '教务处通知公告',
-                docs: 'https://docs.rsshub.app/university.html#bei-jing-lin-ye-da-xue',
+                docs: 'https://docs.rsshub.app/routes/university#bei-jing-lin-ye-da-xue',
                 source: '/:type/index.html',
                 target: '/bjfu/jwc/:type',
             },
@@ -28,7 +28,7 @@ module.exports = {
         kyc: [
             {
                 title: '科技处通知公告',
-                docs: 'https://docs.rsshub.app/university.html#bei-jing-lin-ye-da-xue',
+                docs: 'https://docs.rsshub.app/routes/university#bei-jing-lin-ye-da-xue',
                 source: '/',
                 target: '/bjfu/kjc',
             },
@@ -36,7 +36,7 @@ module.exports = {
         news: [
             {
                 title: '绿色新闻网',
-                docs: 'https://docs.rsshub.app/university.html#bei-jing-lin-ye-da-xue',
+                docs: 'https://docs.rsshub.app/routes/university#bei-jing-lin-ye-da-xue',
                 source: '/:type/index.html',
                 target: '/bjfu/news/:type',
             },

package/lib/v2/bjp/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '每日一图',
-                docs: 'https://docs.rsshub.app/picture.html#bei-jing-tian-wen-guan',
+                docs: 'https://docs.rsshub.app/routes/picture#bei-jing-tian-wen-guan',
                 source: ['/APOD/today.shtml', '/APOD/list.shtml', '/'],
                 target: '/bjp/apod',
             },

package/lib/v2/bjsk/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '通用',
-                docs: 'https://docs.rsshub.app/government.html#bei-jing-she-ke-wang',
+                docs: 'https://docs.rsshub.app/routes/government#bei-jing-she-ke-wang',
                 source: ['/*'],
                 target: (_, url) => `/bjsk/${url.split('/')[3].replace('.html', '')}`,
             },
@@ -12,7 +12,7 @@ module.exports = {
         keti: [
             {
                 title: '基金项目管理平台',
-                docs: 'https://docs.rsshub.app/government.html#bei-jing-she-ke-wang',
+                docs: 'https://docs.rsshub.app/routes/government#bei-jing-she-ke-wang',
                 source: ['/indexAction!to_index.action', '/'],
                 target: '/bjsk/keti/:id',
             },

package/lib/v2/bjwxdxh/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         www: [
             {
                 title: '最新资讯',
-                docs: 'https://docs.rsshub.app/government.html#bei-jing-wu-xian-dian-xie-hui',
+                docs: 'https://docs.rsshub.app/routes/government#bei-jing-wu-xian-dian-xie-hui',
                 source: '/news/class/',
                 target: (params, url) => (url.includes('?') ? `/bjwxdxh/${url.split('?')[1].split('.')[0]}` : '/bjwxdxh'),
             },

package/lib/v2/bjx/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         guangfu: [
             {
                 title: '光伏 - 分类',
-                docs: 'https://docs.rsshub.app/traditional-media.html#bei-ji-xing-dian-li-wang',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#bei-ji-xing-dian-li-wang',
                 source: ['/:type', '/'],
                 target: '/bjx/gf/:type?',
             },
@@ -12,7 +12,7 @@ module.exports = {
         huanbao: [
             {
                 title: '环保要闻',
-                docs: 'https://docs.rsshub.app/traditional-media.html#bei-ji-xing-dian-li-wang',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#bei-ji-xing-dian-li-wang',
                 source: ['/yw', '/'],
                 target: '/bjx/huanbao',
             },

package/lib/v2/blizzard/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:language?/:category?': ['nczitzk'],
+};

package/lib/{routes → v2}/blizzard/news.js

@@ -5,37 +5,36 @@ module.exports = async (ctx) => {
     const category = ctx.params.category || '';
     const language = ctx.params.language || 'en-us';
 
-    const rootUrl = `https://${language === 'zh-cn' ? 'cn.' : ''}news.blizzard.com`;
+    const rootUrl = 'https://news.blizzard.com';
     const currentUrl = `${rootUrl}/${language}/${category}`;
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
+    const apiUrl = `${rootUrl}/${language}/blog/list`;
+    const response = await got(apiUrl, {
+        searchParams: {
+            community: category === '' ? 'all' : category,
+        },
     });
 
-    const $ = cheerio.load(response.data);
+    const $ = cheerio.load(response.data.html, null, false);
 
-    const list = $('.FeaturedArticle-text a, .ArticleListItem article a')
-        .slice(0, 10)
-        .map((_, item) => {
+    const list = $('.FeaturedArticle-text > a, .ArticleListItem > article > a')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30)
+        .toArray()
+        .map((item) => {
             item = $(item);
             return {
                 title: item.text(),
                 link: `${rootUrl}${item.attr('href')}`,
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
+                const detailResponse = await got(item.link);
                 const content = cheerio.load(detailResponse.data);
 
                 item.author = content('.ArticleDetail-bylineAuthor').text();
-                item.description = content('.ArticleDetail-content').html();
+                item.description = content('.ArticleDetail-headingImageBlock').html() + content('.ArticleDetail-content').html();
                 item.pubDate = content('.ArticleDetail-subHeadingLeft time').attr('timestamp');
 
                 return item;

package/lib/v2/blizzard/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'blizzard.com': {
+        _name: 'Blizzard',
+        news: [
+            {
+                title: 'News',
+                docs: 'https://docs.rsshub.app/routes/game#blizzard',
+                source: ['/:language/:category?', '/:language'],
+                target: '/blizzard/news/:language/:category?',
+            },
+        ],
+    },
+};

package/lib/v2/blizzard/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/news/:language?/:category?', require('./news'));
+};