rsshub 1.0.0-master.ff97759 → 1.0.0-master.ffea261

package/lib/v2/gov/miit/zcjd.js

@@ -1,53 +1,62 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const baseUrl = 'https://www.miit.gov.cn';
+const siteUrl = `${baseUrl}/zwgk/zcjd/index.html`;
 
 module.exports = async (ctx) => {
-    const base_url = 'http://www.miit.gov.cn/n1146295/n7281315/';
-    const response = await got.get(base_url);
+    const response = await got(siteUrl);
     const $ = cheerio.load(response.data);
-    const list = $('.clist_con li').get();
-
-    const ProcessFeed = (data) => {
-        const $ = cheerio.load(data);
-
-        const content = $('p');
-        return content.html();
-    };
-
-    const items = await Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const $a = $('a');
-            let link = $a.attr('href');
-            if (link.startsWith('..')) {
-                link = base_url + link;
-            }
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const response = await got({
-                method: 'get',
-                url: link,
-            });
-
-            const single = {
-                title: $a.text(),
-                description: ProcessFeed(response.data),
-                link,
+    const buildStatic = $('script[parseType=buildstatic]');
+    const requestUrl = buildStatic.attr('url');
+    const queryData = JSON.parse(buildStatic.attr('querydata').replace(/'/g, '"'));
+
+    const { data } = await got(`${baseUrl}${requestUrl}`, {
+        headers: {
+            referer: siteUrl,
+        },
+        searchParams: queryData,
+    });
+    const list = cheerio.load(data.data.html, null, false);
+
+    let items = list('.page-content ul li')
+        .toArray()
+        .map((item) => {
+            item = list(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: new URL(item.find('a').attr('href'), baseUrl).href,
+                pubDate: parseDate(item.find('.fr').text(), 'YYYY-MM-DD'),
             };
-
-            ctx.cache.set(link, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+
+                $('iframe').each((_, e) => {
+                    e = $(e);
+                    if (e.attr('src').startsWith('/')) {
+                        e.attr('src', new URL(e.attr('src'), baseUrl).href);
+                    }
+                });
+                item.author = $('.cinfo')
+                    .text()
+                    .match(/来源：(.*)/)[1];
+                item.pubDate = timezone(parseDate($('#con_time').text(), 'YYYY-MM-DD HH:mm'), +8);
+                item.description = $('.ccontent').html();
+
+                return item;
+            })
+        )
     );
 
     ctx.state.data = {
-        title: '中国工业化和信息部',
-        link: 'http://www.miit.gov.cn',
-        description: '政策解读',
+        title: `${$('head title').text()} - ${$('meta[name=SiteName]').attr('content')}`,
+        link: siteUrl,
         item: items,
     };
 };

package/lib/v2/gov/moe/moe.js

@@ -3,13 +3,13 @@ const cheerio = require('cheerio');
 const logger = require('@/utils/logger');
 const { parseDate } = require('@/utils/parse-date');
 
-const moeUrl = 'http://www.moe.gov.cn/';
+const moeUrl = 'https://www.moe.gov.cn/';
 const typesIdMap = [
     { type: 'policy_anal', id: 'tt_con2', name: '政策解读' },
     { type: 'newest_file', id: 'nine_con1', name: '最新文件' },
     { type: 'notice', id: 'nine_con2', name: '公告公示' },
     { type: 'edu_ministry_news', id: 'nine_con3', name: '教育部简报' },
-    { type: 'edu_focus_news', id: 'eight_con2 .TRS_Editor', name: '教育要闻' },
+    { type: 'edu_focus_news', id: 'eight_con2 .pchide>.TRS_Editor', name: '教育要闻' },
 ];
 
 module.exports = async (ctx) => {
@@ -45,27 +45,39 @@ module.exports = async (ctx) => {
                     const firstA = item.find('a');
                     const itemUrl = new URL(firstA.attr('href'), moeUrl).href;
 
-                    const infos = await ctx.cache.tryGet(itemUrl, async () => {
-                        const res = {};
-                        const response = await got(itemUrl);
-                        const data = cheerio.load(response.data);
+                    let infos;
 
-                        if (itemUrl.indexOf('www.gov.cn') > -1) {
-                            res.description = data('div.pages_content > p').html();
-                            res.title = data('div.article > h1').text();
-                        } else if (itemUrl.indexOf('srcsite') > -1) {
-                            res.description = data('div#xxgk_content_div').html();
-                            res.title = data('div#xxgk_content_redheadbg > h1').text();
-                        } else if (itemUrl.indexOf('jyb_') > -1) {
-                            res.description = data('div.TRS_Editor').html();
-                            res.title = data('div.TRS_Editor').siblings('h1').first().text();
-                        }
+                    // some live pages have no content, just return the liva page url
+                    if (itemUrl.includes('/live/')) {
+                        infos = {
+                            description: firstA.html(),
+                        };
+                    } else {
+                        infos = await ctx.cache.tryGet(itemUrl, async () => {
+                            const res = {};
+                            const response = await got({
+                                method: 'get',
+                                url: itemUrl,
+                                headers: {
+                                    Referer: moeUrl,
+                                },
+                            });
+                            const data = cheerio.load(response.data);
 
-                        return res;
-                    });
+                            if (itemUrl.includes('www.gov.cn')) {
+                                res.description = data('#UCAP-CONTENT').html();
+                            } else if (itemUrl.includes('srcsite')) {
+                                res.description = data('div#content_body_xxgk').html();
+                            } else if (itemUrl.includes('jyb_')) {
+                                res.description = data('div.moe-detail-box').html() || data('div#moe-detail-box').html();
+                            }
+
+                            return res;
+                        });
+                    }
 
                     return {
-                        title: infos.title,
+                        title: firstA.text(),
                         description: infos.description,
                         link: itemUrl,
                         pubDate: parseDate(item.find('span').text(), 'MM-DD'),

package/lib/v2/gov/mofcom/article.js

@@ -0,0 +1,68 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const host = 'http://www.mofcom.gov.cn';
+
+module.exports = async (ctx) => {
+    const suffix = ctx.params.suffix;
+    const url = `http://www.mofcom.gov.cn/article/${suffix}/`;
+    const { data: res } = await got(url);
+    const $ = cheerio.load(res);
+    $('.listline').remove();
+    const list = $('.txtList_01 li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const date =
+                item.find('span').length !== 0
+                    ? item
+                          .find('span')
+                          .text()
+                          .match(/((\d{4}-\d{2}-\d{2})(\s\d{2}:\d{2}:\d{2})?)/)[1]
+                    : item
+                          .find('a')
+                          .attr('title')
+                          .match(/(\d{4}年\d{1,2}月\d{1,2})/)[1];
+            return {
+                title: item.find('a').attr('title'),
+                link: host + item.find('a').attr('href'),
+                pubDate: timezone(parseDate(date, ['YYYY-MM-DD HH:mm:ss', 'YYYY-MM-DD', 'YYYY年M月D']), 8),
+            };
+        });
+
+    await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                let responses = await got(item.link);
+                // xwfb/xwlxfbh || xwfb/xwztfbh
+                const redirect = responses.data.match(/_cofing1=\{href:"(.*)",type/) || responses.data.match(/window\.location\.href='(.*)'/);
+                if (redirect) {
+                    responses = await got(redirect[1], {
+                        headers: {
+                            Referer: item.link,
+                        },
+                    });
+                }
+                const $ = cheerio.load(responses.data);
+                const iframe = $('.art-con iframe').attr('src');
+                if ($('.art-con iframe').attr('src')) {
+                    const { data: res } = await got(iframe);
+                    item.description = cheerio.load(res)('tbody').html();
+                } else {
+                    item.description = $('.art-con').html() || /* xwfb/xwztfbh */ $('.textlive').html();
+                }
+                item.pubDate = $('meta[name="PubDate"]').length ? timezone(parseDate($('meta[name="PubDate"]').attr('content'), 'YYYY-MM-DD HH:mm'), 8) : item.pubDate;
+
+                return item;
+            })
+        )
+    );
+    ctx.state.data = {
+        title: $('head > title').text(),
+        description: $('meta[name=description]').attr('content'),
+        link: url,
+        item: list,
+    };
+};

package/lib/v2/gov/moj/aac/news.js

@@ -0,0 +1,49 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const baseUrl = 'https://www.aac.moj.gov.tw';
+
+module.exports = async (ctx) => {
+    const { type } = ctx.params;
+    const url = `${baseUrl}/7204/7246/?Page=1&PageSize=40${type ? `&type=${type}` : ''}`;
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    $('.num').remove();
+    const list = $('.list ul li a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const isDownload = /檔案下載/.test(item.attr('title'));
+            const title = isDownload ? item.text().trim() : item.attr('title');
+            return {
+                title,
+                link: new URL(item.attr('href'), baseUrl).href,
+                isDownload,
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (!item.isDownload) {
+                    const response = await got(item.link);
+                    const $ = cheerio.load(response.data);
+
+                    item.pubDate = timezone(parseDate($('.info time').attr('datetime'), 'YYYY-MM-DD HH:mm:ss'), +8);
+                    $('.info, button').remove();
+                    item.description = $('.cp').html() + ($('.lightbox_slider').length ? $('.lightbox_slider').html() : '') + ($('.file_download').length ? $('.file_download').html() : '');
+                }
+                delete item.isDownload;
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        item: items,
+        language: 'zh-TW',
+    };
+};

package/lib/v2/gov/nifdc/index.js

@@ -0,0 +1,86 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const params = ctx.path === '/nifdc' ? '/nifdc/bshff/ylqxbzhgl/qxggtzh' : ctx.path;
+
+    const rootUrl = 'https://www.nifdc.org.cn';
+    const currentUrl = `${rootUrl}${params}/index.html`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.list ul li a')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href');
+
+            return {
+                title: item.text(),
+                link: /^http/.test(link) ? link : new URL(link, currentUrl).href,
+                pubDate: parseDate(
+                    item
+                        .next()
+                        .text()
+                        .match(/\((.*)\)/)
+                ),
+            };
+        });
+
+    const browser = await require('@/utils/puppeteer')();
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (/^https:\/\/www\.nmpa\.gov\.cn\//.test(item.link)) {
+                    const page = await browser.newPage();
+                    await page.setRequestInterception(true);
+
+                    page.on('request', (request) => {
+                        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                    });
+
+                    await page.goto(item.link, {
+                        waitUntil: 'domcontentloaded',
+                    });
+                    await page.waitForSelector('.text');
+                    const html = await page.evaluate(() => document.documentElement.innerHTML);
+                    await page.close();
+
+                    const content = cheerio.load(html);
+
+                    item.description = content('.text').html();
+                } else {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+
+                    const content = cheerio.load(detailResponse.data);
+
+                    item.description = content('.text').append(content('.fujian')).html();
+                    item.pubDate = timezone(parseDate(content('meta[name="PubDate"]').attr('content')), +8);
+                }
+
+                return item;
+            })
+        )
+    );
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: $('title').text().replace(/----/, ' - '),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/gov/nmpa/generic.js

@@ -0,0 +1,83 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+const config = require('@/config').value;
+const baseUrl = 'https://www.nmpa.gov.cn';
+
+module.exports = async (ctx) => {
+    const path = ctx.params[0];
+    const url = `${baseUrl}/${path.endsWith('/') ? path.slice(0, -1) : path}/index.html`;
+    const browser = await require('@/utils/puppeteer')({ stealth: true });
+    const data = await ctx.cache.tryGet(
+        url,
+        async () => {
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
+            await page.goto(url, {
+                waitUntil: 'domcontentloaded',
+            });
+            await page.waitForSelector('.list');
+
+            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            await page.close();
+            const $ = cheerio.load(html);
+
+            return {
+                title: $('head title').text(),
+                description: $('meta[name=ColumnDescription]').attr('content'),
+                items: $('.list ul li')
+                    .toArray()
+                    .map((item) => {
+                        item = $(item);
+                        return {
+                            title: item.find('a').text().trim(),
+                            link: new URL(item.find('a').attr('href'), baseUrl).href,
+                            pubDate: parseDate(item.find('span').text(), 'YYYY-MM-DD'),
+                        };
+                    }),
+            };
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const items = await Promise.all(
+        data.items.map((item) => {
+            if (/^https:\/\/www\.nmpa\.gov\.cn\//.test(item.link)) {
+                return ctx.cache.tryGet(item.link, async () => {
+                    const page = await browser.newPage();
+                    await page.setRequestInterception(true);
+                    page.on('request', (request) => {
+                        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                    });
+                    await page.goto(item.link, {
+                        waitUntil: 'domcontentloaded',
+                    });
+                    await page.waitForSelector('.text');
+
+                    const html = await page.evaluate(() => document.documentElement.innerHTML);
+                    await page.close();
+                    const $ = cheerio.load(html);
+                    item.description = $('.text').html();
+                    return item;
+                });
+            } else if (/^https:\/\/mp\.weixin\.qq\.com\//.test(item.link)) {
+                return finishArticleItem(ctx, item);
+            } else {
+                return item;
+            }
+        })
+    );
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: data.title,
+        description: data.description,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/gov/nopss/index.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const iconv = require('iconv-lite');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const params = ctx.path === '/nopss' ? '/GB/219469' : ctx.path.replace(/^\/nopss/, '');
+
+    const rootUrl = 'http://www.nopss.gov.cn';
+    const currentUrl = `${rootUrl}${params}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+        responseType: 'buffer',
+    });
+
+    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+
+    let items = $('.p2j_list_con .clearfix li a')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 40)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: `${rootUrl}${item.attr('href')}`,
+                pubDate: timezone(parseDate(item.next().text(), '[YYYY-MM-DD HH:mm]'), +8),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                    responseType: 'buffer',
+                });
+
+                const content = cheerio.load(iconv.decode(detailResponse.data, 'gbk'));
+
+                item.description = content('.text_con').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/gov/nrta/news.js

@@ -25,7 +25,6 @@ module.exports = async (ctx) => {
     const data = response.data.replace(regex, '$1');
 
     const $ = cheerio.load(data, {
-        normalizeWhitespace: true,
         xmlMode: true,
     });
 

package/lib/v2/gov/nsfc/index.js

@@ -0,0 +1,83 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+const types = {
+    jjyw: {
+        title: '基金要闻',
+        url: '/publish/portal0/tab440/',
+    },
+    tzgg: {
+        title: '通知公告',
+        url: '/publish/portal0/tab442/',
+    },
+    zzcg: {
+        title: '资助成果',
+        url: '/publish/portal0/tab448/',
+    },
+    kpkx: {
+        title: '科普快讯',
+        url: '/publish/portal0/tab446/',
+    },
+};
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type ?? 'jjyw';
+
+    const rootUrl = 'https://www.nsfc.gov.cn';
+    const currentUrl = `${rootUrl}${types[type].url}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+        https: {
+            rejectUnauthorized: false,
+        },
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.fl a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href');
+
+            return {
+                title: item.attr('title'),
+                link: `${/^http/.test(link) ? '' : rootUrl}${link}`,
+                pubDate: parseDate(item.parent().next().text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (/nsfc\.gov\.cn/.test(item.link)) {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                        https: {
+                            rejectUnauthorized: false,
+                        },
+                    });
+
+                    const content = cheerio.load(detailResponse.data);
+
+                    item.description = content('.content_xilan').html();
+                    item.pubDate = timezone(parseDate(content('meta[name="docdate"]').attr('content')), +8);
+                }
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `国家自然科学基金委员会 - ${$('title').text()}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/gov/pbc/goutongjiaoliu.js

@@ -5,22 +5,25 @@ const timezone = require('@/utils/timezone');
 module.exports = async (ctx) => {
     const link = 'http://www.pbc.gov.cn/goutongjiaoliu/113456/113469/index.html';
 
-    const browser = await require('@/utils/puppeteer')();
+    const browser = await require('@/utils/puppeteer')({ stealth: true });
     const page = await browser.newPage();
-    await page.goto(link);
-    const html = await page.evaluate(
-        () =>
-            // eslint-disable-next-line
-            document.querySelector('body').innerHTML
-    );
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(link, {
+        waitUntil: 'domcontentloaded',
+    });
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+
     const $ = cheerio.load(html);
     const list = $('font.newslist_style')
         .map((_, item) => {
             item = $(item);
             const a = item.find('a[title]');
             return {
-                title: a.text(),
-                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn'),
+                title: a.attr('title'),
+                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn').href,
             };
         })
         .get();
@@ -29,12 +32,14 @@ module.exports = async (ctx) => {
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailPage = await browser.newPage();
-                await detailPage.goto(item.link);
-                const detailHtml = await detailPage.evaluate(
-                    () =>
-                        // eslint-disable-next-line
-                        document.querySelector('body').innerHTML
-                );
+                await detailPage.setRequestInterception(true);
+                detailPage.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await detailPage.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
+                });
+                const detailHtml = await detailPage.evaluate(() => document.documentElement.innerHTML);
                 const content = cheerio.load(detailHtml);
                 item.description = content('#zoom').html();
                 item.pubDate = timezone(parseDate(content('.hui12').eq(5).text()), +8);