rsshub 1.0.0-master.fe69819 → 1.0.0-master.fe775f5

package/lib/v2/gov/miit/yjzj.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const rootUrl = 'https://www.miit.gov.cn';
+
+module.exports = async (ctx) => {
+    const url = `${rootUrl}/gzcy/yjzj/index.html`;
+
+    const cookieResponse = await got(url);
+    const cookie = cookieResponse.headers['set-cookie'][0].split(';')[0];
+    const indexContent = cheerio.load(cookieResponse.data);
+    const dataRequestUrl = indexContent('div.clist_con > script:nth-child(2)')
+        .map((_, item) => ({
+            url: `${rootUrl}${indexContent(item).attr('url')}`,
+            queryData: JSON.parse(indexContent(item).attr('querydata').replace(/"/g, '|').replace(/'/g, '"').replace(/\|/g, '"')),
+        }))
+        .get()[0];
+
+    const dataUrl = `${dataRequestUrl.url}?${Object.keys(dataRequestUrl.queryData)
+        .map((key) => `${key}=${dataRequestUrl.queryData[key]}`)
+        .join('&')}`;
+    const response = await got({
+        method: 'get',
+        url: dataUrl,
+        headers: {
+            Cookie: cookie,
+        },
+    });
+    const $ = cheerio.load(response.data.data.html);
+    const list = $('ul > li')
+        .map((_, item) => ({
+            title: $(item).find('a').text(),
+            link: new URL($(item).find('a').attr('href'), rootUrl).href,
+            pubDate: parseDate($(item).find('span').text(), 'YYYY-MM-DD'),
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('#con_con')
+                    .html()
+                    .replace(/(<iframe.*?src=")(.*?)(".*?>)/g, '$1' + rootUrl + '$2' + '$3');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `工业和信息化部 - 意见征集`,
+        link: url,
+        item: items,
+    };
+};

package/lib/{routes → v2/gov}/pbc/goutongjiaoliu.js

@@ -1,8 +1,9 @@
-const url = require('url');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
-    const link = `http://www.pbc.gov.cn/goutongjiaoliu/113456/113469/index.html`;
+    const link = 'http://www.pbc.gov.cn/goutongjiaoliu/113456/113469/index.html';
 
     const browser = await require('@/utils/puppeteer')();
     const page = await browser.newPage();
@@ -19,8 +20,7 @@ module.exports = async (ctx) => {
             const a = item.find('a[title]');
             return {
                 title: a.text(),
-                link: url.resolve(`http://www.pbc.gov.cn`, a.attr('href')),
-                pubDate: new Date(item.next('span.hui12').text()).toUTCString(),
+                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn'),
             };
         })
         .get();
@@ -37,6 +37,7 @@ module.exports = async (ctx) => {
                 );
                 const content = cheerio.load(detailHtml);
                 item.description = content('#zoom').html();
+                item.pubDate = timezone(parseDate(content('.hui12').eq(5).text()), +8);
                 return item;
             })
         )

package/lib/v2/gov/pbc/gzlw.js

@@ -0,0 +1,27 @@
+const { processItems } = require('./utils');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const host = 'http://www.pbc.gov.cn';
+
+module.exports = async (ctx) => {
+    const url = `${host}/redianzhuanti/118742/4122386/4122692/index.html`;
+
+    const response = await got.post(url);
+    const $ = cheerio.load(response.data);
+    const list = $('li.clearfix')
+        .map((_index, item) => ({
+            title: $(item).find('a').text(),
+            link: new URL($(item).find('a').attr('href'), host).href,
+            author: $(item).find('span.fr').text().replace(/…/g, ''),
+        }))
+        .get();
+
+    const items = await processItems(list, ctx);
+
+    ctx.state.data = {
+        title: '中国人民银行 工作论文',
+        link: url,
+        item: items,
+    };
+};

package/lib/{routes → v2/gov}/pbc/tradeAnnouncement.js

@@ -1,8 +1,9 @@
-const url = require('url');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
-    const link = `http://www.pbc.gov.cn/zhengcehuobisi/125207/125213/125431/125475/index.html`;
+    const link = 'http://www.pbc.gov.cn/zhengcehuobisi/125207/125213/125431/125475/index.html';
 
     const browser = await require('@/utils/puppeteer')();
     const page = await browser.newPage();
@@ -19,8 +20,7 @@ module.exports = async (ctx) => {
             const a = item.find('a[title]');
             return {
                 title: a.text(),
-                link: url.resolve(`http://www.pbc.gov.cn`, a.attr('href')),
-                pubDate: new Date(item.next('span.hui12').text()).toUTCString(),
+                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn'),
             };
         })
         .get();
@@ -37,6 +37,7 @@ module.exports = async (ctx) => {
                 );
                 const content = cheerio.load(detailHtml);
                 item.description = content('#zoom').html();
+                item.pubDate = timezone(parseDate(content('#shijian').text()), +8);
                 return item;
             })
         )

package/lib/v2/gov/pbc/utils.js

@@ -0,0 +1,20 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const processItems = async (list, ctx) =>
+    await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got.post(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('div.xxy_text').html();
+
+                return item;
+            })
+        )
+    );
+
+module.exports = {
+    processItems,
+};

package/lib/v2/gov/pbc/zcyj.js

@@ -0,0 +1,29 @@
+const { processItems } = require('./utils');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const host = 'http://www.pbc.gov.cn';
+
+module.exports = async (ctx) => {
+    const url = `${host}/redianzhuanti/118742/4122386/4122510/index.html`;
+
+    const response = await got.post(url);
+    const $ = cheerio.load(response.data);
+    const list = $('li.clearfix')
+        .map((_index, item) => ({
+            title: $(item).find('a').text(),
+            link: new URL($(item).find('a').attr('href'), host).href,
+            pubDate: timezone(parseDate($(item).find('span.fr').text(), 'YYYY-MM-DD'), +8),
+        }))
+        .get();
+
+    const items = await processItems(list, ctx);
+
+    ctx.state.data = {
+        title: '中国人民银行 政策研究',
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/gov/radar.js

@@ -376,4 +376,61 @@ module.exports = {
             },
         ],
     },
+    'miit.gov.cn': {
+        _name: '工业和信息化部',
+        '.': [
+            {
+                title: '部门 文件发布',
+                docs: 'https://docs.rsshub.app/government.html#zhong-guo-gong-ye-he-xin-xi-hua-bu',
+                source: ['/jgsj/:ministry/wjfb/index.html'],
+                target: '/miit/wjfb/:ministry',
+            },
+            {
+                title: '征集意见',
+                docs: 'https://docs.rsshub.app/government.html#zhong-guo-gong-ye-he-xin-xi-hua-bu',
+                source: ['/gzcy/yjzj/index.html'],
+                target: '/miit/yjzj',
+            },
+        ],
+    },
+    'mee.gov.cn': {
+        _name: '生态环境部',
+        www: [
+            {
+                title: '要闻动态',
+                docs: 'https://docs.rsshub.app/government.html#zhong-hua-ren-min-gong-he-guo-sheng-tai-huan-jing-bu',
+                source: ['/ywdt/:category'],
+                target: '/gov/mee/ywdt/:category',
+            },
+        ],
+    },
+    'pbc.gov.cn': {
+        _name: '中国人民银行',
+        '.': [
+            {
+                title: '沟通交流',
+                docs: 'https://docs.rsshub.app/finance.html#zhong-guo-ren-min-yin-xing',
+                source: ['/goutongjiaoliu/113456/113469/index.html'],
+                target: '/gov/pbc/goutongjiaoliu',
+            },
+            {
+                title: '货币政策司公开市场交易公告',
+                docs: 'https://docs.rsshub.app/finance.html#zhong-guo-ren-min-yin-xing',
+                source: ['/zhengcehuobisi/125207/125213/125431/125475/index.html'],
+                target: '/gov/pbc/zhengcehuobisi',
+            },
+            {
+                title: '政策研究',
+                docs: 'https://docs.rsshub.app/finance.html#zhong-guo-ren-min-yin-xing',
+                source: ['/redianzhuanti/118742/4122386/4122510/index.html'],
+                target: '/gov/pbc/zcyj',
+            },
+            {
+                title: '工作论文',
+                docs: 'https://docs.rsshub.app/finance.html#zhong-guo-ren-min-yin-xing',
+                source: ['/redianzhuanti/118742/4122386/4122692/index.html'],
+                target: '/gov/pbc/gzlw',
+            },
+        ],
+    },
 };

package/lib/v2/gov/router.js

@@ -8,4 +8,14 @@ module.exports = function (router) {
     router.get('/guangdong/tqyb/sncsyjxh', require('./guangdong/tqyb/sncsyjxh'));
     router.get('/huizhou/zwgk/:category?', require('./huizhou/zwgk/index'));
     router.get('/sichuan/deyang/govpulicinfo/:countyName/:institutionName?', require('./sichuan/deyang/govpulicinfo'));
+    router.get('/miit/wjfb/:ministry', require('./miit/wjfb'));
+    router.get('/miit/yjzj', require('./miit/yjzj'));
+    router.get('/miit/zcwj', require('./miit/zcwj'));
+    router.get('/miit/wjgs', require('./miit/wjgs'));
+    router.get('/miit/zcjd', require('./miit/zcjd'));
+    router.get('/mee/ywdt/:category?', require('./mee/ywdt'));
+    router.get('/pbc/goutongjiaoliu', require('./pbc/goutongjiaoliu'));
+    router.get('/pbc/tradeAnnouncement', require('./pbc/tradeAnnouncement'));
+    router.get('/pbc/gzlw', require('./pbc/gzlw'));
+    router.get('/pbc/zcyj', require('./pbc/zcyj'));
 };

package/lib/{routes → v2}/guancha/headline.js

@@ -1,9 +1,12 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const rootUrl = 'https://www.guancha.cn';
     const currentUrl = `${rootUrl}/GuanChaZheTouTiao/list_1.shtml`;
+
     const response = await got({
         method: 'get',
         url: currentUrl,
@@ -11,20 +14,20 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('.headline-list li .content-headline h3 a')
-        .map((_, item) => {
+    let items = $('.headline-list li .content-headline h3 a')
+        .toArray()
+        .map((item) => {
             item = $(item);
             return {
                 title: item.text(),
-                link: `${rootUrl}${item.attr('href')}`,
                 description: item.parent().next().html(),
-                pubDate: new Date(item.parents('div').eq(0).find('span').text()).toUTCString(),
+                link: `${rootUrl}${item.attr('href').replace(/\.shtml$/, '_s.shtml')}`,
+                pubDate: timezone(parseDate(item.parents('div').first().find('span').text()), +8),
             };
-        })
-        .get();
+        });
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',

package/lib/{routes → v2}/guancha/index.js

@@ -1,21 +1,42 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const date = require('@/utils/date');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { parseRelativeDate } = require('@/utils/parse-date');
 
 const config = {
-    review: { title: '评论 & 研究', query: '.module-news-main' },
-    story: { title: '要闻', query: '.img-List' },
-    fengwen: { title: '风闻', query: '.fengwen-list' },
-    redian: { title: '热点' },
-    gundong: { title: '滚动' },
-    all: { title: '全部' },
-    home: { title: '首页' },
-    others: { title: '热点 & 滚动' },
+    review: {
+        title: '评论 & 研究',
+        query: '.module-news-main',
+    },
+    story: {
+        title: '要闻',
+        query: '.img-List',
+    },
+    fengwen: {
+        title: '风闻',
+        query: '.fengwen-list',
+    },
+    redian: {
+        title: '热点',
+    },
+    gundong: {
+        title: '滚动',
+    },
+    all: {
+        title: '全部',
+    },
+    home: {
+        title: '首页',
+    },
+    others: {
+        title: '热点 & 滚动',
+    },
 };
 
 module.exports = async (ctx) => {
     const total = 10;
-    const caty = ctx.params.caty || 'all';
+    const category = ctx.params.category ?? 'all';
     const rootUrl = 'https://www.guancha.cn';
 
     let newsList = [],
@@ -24,7 +45,7 @@ module.exports = async (ctx) => {
 
     // 'review', 'story' and 'fengwen' come from homepage.
 
-    if (caty === 'review' || caty === 'story' || caty === 'fengwen' || caty === 'all' || caty === 'home') {
+    if (category === 'review' || category === 'story' || category === 'fengwen' || category === 'all' || category === 'home') {
         const response = await got({
             method: 'get',
             url: rootUrl,
@@ -35,54 +56,59 @@ module.exports = async (ctx) => {
         const fetchPost = (slice) =>
             slice
                 .find('h4.module-title a')
+                .toArray()
 
                 // Filter some blank links which lead to no contents but 'https://user.guancha.cn'.
 
-                .filter((_, item) => $(item).attr('href') !== 'https://user.guancha.cn')
-                .map((_, item) => {
+                .filter((item) => $(item).attr('href') !== 'https://user.guancha.cn')
+                .map((item) => {
                     item = $(item);
+
                     const link = item.attr('href');
+
                     return {
                         title: item.text(),
-                        link: (link.indexOf('http') === 0 ? '' : rootUrl) + link,
+                        link: `${link.indexOf('http') === 0 ? '' : rootUrl}${link.replace(/\.shtml/, '_s.shtml')}`,
                     };
-                })
-                .get();
+                });
 
-        if (caty === 'all' || caty === 'home') {
+        if (category === 'all' || category === 'home') {
             newsList = fetchPost($(config.review.query))
                 .slice(0, total / 3)
                 .concat(fetchPost($(config.story.query)).slice(0, total / 3), fetchPost($(config.fengwen.query)).slice(0, total / 3));
         } else {
-            newsList = fetchPost($(config[caty].query)).slice(0, total);
+            newsList = fetchPost($(config[category].query)).slice(0, total);
         }
     }
 
     // 'redian' and 'gundong' come from api.
 
-    if (caty === 'redian' || caty === 'all' || caty === 'others') {
+    if (category === 'redian' || category === 'all' || category === 'others') {
         const response = await got({
             method: 'get',
             url: `${rootUrl}/api/redian.htm`,
         });
+
         redianList = response.data.items
             .map((item) => ({
                 title: item.TITLE,
-                link: `${rootUrl}${item.HTTP_URL}`,
+                link: `${rootUrl}${item.HTTP_URL.replace(/\.shtml/, '_s.shtml')}`,
             }))
-            .slice(0, caty === 'all' ? total / 3 : total);
+            .slice(0, category === 'all' ? total / 3 : total);
     }
-    if (caty === 'gundong' || caty === 'all' || caty === 'others') {
+
+    if (category === 'gundong' || category === 'all' || category === 'others') {
         const response = await got({
             method: 'get',
             url: `${rootUrl}/api/gundong.htm`,
         });
+
         gundongList = response.data.items
             .map((item) => ({
                 title: item.TITLE,
-                link: `${rootUrl}${item.HTTP_URL}`,
+                link: `${rootUrl}${item.HTTP_URL.replace(/\.shtml/, '_s.shtml')}`,
             }))
-            .slice(0, caty === 'all' ? total / 3 : total);
+            .slice(0, category === 'all' ? total / 3 : total);
     }
 
     const items = await Promise.all(
@@ -98,8 +124,9 @@ module.exports = async (ctx) => {
                 //  => https://user.guancha.cn/main/content?id=399176
 
                 const jumpMatch = detailResponse.data.match(/user.guancha.cn\/main\/content\?id=(.*)";/);
+
                 if (jumpMatch !== null) {
-                    item.link = `https://user.guancha.cn/main/content?id=${jumpMatch[1]}`;
+                    item.link = `https://user.guancha.cn/main/content?id=${jumpMatch[1]}&page=0`;
                     detailResponse = await got({
                         method: 'get',
                         url: item.link,
@@ -112,9 +139,9 @@ module.exports = async (ctx) => {
                 if (dateMatch === null) {
                     // PubDates of posts in 'fengwen' are in an informal format.
 
-                    item.pubDate = date(content('.time1').text(), 8);
+                    item.pubDate = parseRelativeDate(content('.time1').text());
                 } else {
-                    item.pubDate = new Date(dateMatch[1]).toUTCString();
+                    item.pubDate = timezone(parseDate(dateMatch[1]), +8);
                 }
 
                 item.description = content('.all-txt').html() || content('.article-txt-content').html();
@@ -125,7 +152,7 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: `观察者网 - ${config[caty].title}`,
+        title: `观察者网 - ${config[category].title}`,
         link: rootUrl,
         item: items,
     };

package/lib/v2/guancha/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/:category?': ['nczitzk', 'Jeason0228'],
+    '/headline': ['nczitzk'],
+    '/member/:category?': ['nczitzk'],
+    '/personalpage/:uid': ['Jeason0228'],
+    '/topic/:id/:order?': ['occupy5', 'nczitzk'],
+};

package/lib/{routes → v2}/guancha/member.js

@@ -1,6 +1,8 @@
 const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
 
-const title = {
+const titles = {
     recommend: '精选',
     books: '观书堂',
     courses: '在线课',
@@ -8,10 +10,11 @@ const title = {
 };
 
 module.exports = async (ctx) => {
-    const caty = ctx.params.caty || 'recommend';
+    const category = ctx.params.category ?? 'recommend';
 
     const rootUrl = 'https://member.guancha.cn';
     const apiUrl = `${rootUrl}/zaixianke/home`;
+
     const response = await got({
         method: 'get',
         url: apiUrl,
@@ -19,11 +22,11 @@ module.exports = async (ctx) => {
 
     let items;
 
-    switch (caty) {
+    switch (category) {
         case 'books':
             items = response.data.data.books.map((item) => ({
                 title: item.title,
-                link: `${rootUrl}/guanshutang/summary.html?id=${item.id}`,
+                link: `${rootUrl}/guanshutang/summary.html?id=${item.id}&page=0`,
                 description: `<img src="${item.cover}"><p>[${item.audio_time}] ${item.desc_short}</p>`,
                 pubDate: new Date(parseInt(item.cover.split('/').pop().substr(0, 10)) * 1000).toUTCString(),
             }));
@@ -51,20 +54,32 @@ module.exports = async (ctx) => {
             break;
 
         default:
-            items = response.data.data[caty].map((item) => ({
-                title: item.title,
-                link: item.jump_url,
-                author: item.author_name,
-                description: `<img src="${item.big_pic}"><p>${item.summary}</p>`,
-                enclosure_url: item.media_url,
-                enclosure_length: item.media_size,
-                enclosure_type: 'audio/mpeg',
-                pubDate: new Date(item.created_at * 1000).toUTCString(),
-            }));
+            items = response.data.data[category].map((item) => {
+                let timeArray = item.media_time && item.media_time.trim().split(/\D+/, 3);
+                timeArray = timeArray && timeArray.filter((item) => item !== '');
+                let itunes_duration;
+                if (timeArray) {
+                    itunes_duration = 0;
+                    itunes_duration += timeArray.length >= 1 ? parseInt(timeArray.slice(-1)) : 0;
+                    itunes_duration += timeArray.length >= 2 ? parseInt(timeArray.slice(-2)) * 60 : 0;
+                    itunes_duration += timeArray.length >= 3 ? parseInt(timeArray.slice(-3)) * 60 * 60 : 0;
+                }
+                return {
+                    title: item.title,
+                    link: item.jump_url,
+                    author: item.author_name,
+                    description: `<img src="${item.big_pic}"><p>${item.summary}</p>`,
+                    enclosure_url: item.media_url,
+                    enclosure_length: item.media_size,
+                    itunes_duration,
+                    enclosure_type: 'audio/mpeg',
+                    pubDate: isNaN(+item.created_at) ? timezone(parseDate(item.created_at), +8) : parseDate(item.created_at * 1000),
+                };
+            });
     }
 
     ctx.state.data = {
-        title: `观学院 - ${title[caty]}`,
+        title: `观学院 - ${titles[category]}`,
         link: `${rootUrl}/index.html`,
         item: items,
     };

package/lib/v2/guancha/radar.js

@@ -0,0 +1,49 @@
+module.exports = {
+    'guancha.cn': {
+        _name: '观察者网',
+        '.': [
+            {
+                title: '头条',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-tou-tiao',
+                source: ['/GuanChaZheTouTiao', '/'],
+                target: '/guancha/headline',
+            },
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-shou-ye',
+                source: ['/'],
+                target: '/guancha/:category?',
+            },
+            {
+                title: '观学院',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-guan-xue-yuan',
+                source: ['/'],
+                target: '/guancha/:category?',
+            },
+        ],
+        app: [
+            {
+                title: '个人主页文章',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-ge-ren-zhu-ye-wen-zhang',
+                source: ['/user/get-published-list', '/'],
+                target: (params, url) => `/guancha/personalpage/${new URL(url).searchParams.get('uid')}`,
+            },
+        ],
+        member: [
+            {
+                title: '观学院',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-guan-xue-yuan',
+                source: ['/'],
+                target: '/guancha/member/recommend',
+            },
+        ],
+        user: [
+            {
+                title: '风闻话题',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-feng-wen-hua-ti',
+                source: ['/topic/post-list', '/'],
+                target: (params, url) => `/guancha/topic/${new URL(url).searchParams.get('topic_id')}/${new URL(url).searchParams.get('order')}`,
+            },
+        ],
+    },
+};

package/lib/v2/guancha/router.js

@@ -0,0 +1,7 @@
+module.exports = function (router) {
+    router.get('/headline', require('./headline'));
+    router.get('/member/:category?', require('./member'));
+    router.get('/personalpage/:uid', require('./personalpage'));
+    router.get('/topic/:id/:order?', require('./topic'));
+    router.get('/:category?', require('./index'));
+};