rsshub 1.0.0-master.fcb92f0 → 1.0.0-master.fcc0fab

package/lib/v2/globallawreview/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'globallawreview.org': {
+        _name: '环球法律评论',
+        '.': [
+            {
+                title: '期刊',
+                docs: 'https://docs.rsshub.app/routes/journal#huan-qiu-fa-lu-ping-lun',
+                source: ['/Magazine/GetIssueContentList', '/'],
+                target: '/globallawreview',
+            },
+        ],
+    },
+};

package/lib/v2/globallawreview/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./'));
+};

package/lib/v2/gocn/radar.js

@@ -4,13 +4,13 @@ module.exports = {
         '.': [
             {
                 title: '文章',
-                docs: 'https://docs.rsshub.app/programming.html#GoCN',
+                docs: 'https://docs.rsshub.app/routes/programming#GoCN',
                 source: ['/'],
                 target: '/gocn',
             },
             {
                 title: '招聘',
-                docs: 'https://docs.rsshub.app/programming.html#GoCN',
+                docs: 'https://docs.rsshub.app/routes/programming#GoCN',
                 source: ['/'],
                 target: '/gocn/jobs',
             },

package/lib/v2/good/radar.js

@@ -4,7 +4,7 @@ module.exports = {
         '.': [
             {
                 title: '今日要闻',
-                docs: 'https://docs.rsshub.app/new-media.html#good-news-jin-ri-yao-wen',
+                docs: 'https://docs.rsshub.app/routes/new-media#good-news-jin-ri-yao-wen',
                 source: ['/'],
                 target: '/good',
             },

package/lib/v2/google/radar.js

@@ -5,14 +5,14 @@ module.exports = {
             {
                 title: '插件更新',
                 source: '/webstore/detail/:name/:id',
-                docs: 'https://docs.rsshub.app/program-update.html#chrome-wang-shang-ying-yong-dian',
+                docs: 'https://docs.rsshub.app/routes/program-update#chrome-wang-shang-ying-yong-dian',
                 target: '/chrome/webstore/extensions/:id',
             },
         ],
         photos: [
             {
                 title: '相册',
-                docs: 'https://docs.rsshub.app/picture.html#google-xiang-ce',
+                docs: 'https://docs.rsshub.app/routes/picture#google-xiang-ce',
                 source: '/share/*',
                 target: (params, url, document) => {
                     const id = document && document.querySelector('html').innerHTML.match(/photos.app.goo.gl\/(.*?)"/)[1];
@@ -23,7 +23,7 @@ module.exports = {
         sites: [
             {
                 title: 'Sites',
-                docs: 'https://docs.rsshub.app/blog.html#google-sites',
+                docs: 'https://docs.rsshub.app/routes/blog#google-sites',
                 source: ['/site/:id/*', '/site/:id'],
                 target: '/google/sites/:id',
             },
@@ -31,31 +31,31 @@ module.exports = {
         fonts: [
             {
                 title: 'Fonts - Sort by Name',
-                docs: 'https://docs.rsshub.app/font.html#google-fonts',
+                docs: 'https://docs.rsshub.app/routes/font#google-fonts',
                 source: ['/'],
                 target: '/google/fonts/alpha',
             },
             {
                 title: 'Fonts - Sort by Trending',
-                docs: 'https://docs.rsshub.app/font.html#google-fonts',
+                docs: 'https://docs.rsshub.app/routes/font#google-fonts',
                 source: ['/'],
                 target: '/google/fonts/trending',
             },
             {
                 title: 'Fonts - Sort by Most Popular',
-                docs: 'https://docs.rsshub.app/font.html#google-fonts',
+                docs: 'https://docs.rsshub.app/routes/font#google-fonts',
                 source: ['/'],
                 target: '/google/fonts/popularity',
             },
             {
                 title: 'Fonts - Sort by Newest',
-                docs: 'https://docs.rsshub.app/font.html#google-fonts',
+                docs: 'https://docs.rsshub.app/routes/font#google-fonts',
                 source: ['/'],
                 target: '/google/fonts/date',
             },
             {
                 title: 'Fonts - Sort by Number of Styles',
-                docs: 'https://docs.rsshub.app/font.html#google-fonts',
+                docs: 'https://docs.rsshub.app/routes/font#google-fonts',
                 source: ['/'],
                 target: '/google/fonts/style',
             },

package/lib/v2/gov/cac/index.js

@@ -19,8 +19,8 @@ module.exports = async (ctx) => {
                 .toArray()
                 .map((item) => {
                     const href = $(item).attr('href');
-                    if (href && href.startsWith('http://www.cac.gov.cn/')) {
-                        const matchArray = href.match(/http:\/\/www\.cac\.gov\.cn(.*?)\/(A.*?\.htm)/);
+                    if (href && href.match(/(?:http:)?\/\/www\.cac\.gov\.cn(.*?)\/(A.*?\.htm)/)) {
+                        const matchArray = href.match(/(?:http:)?\/\/www\.cac\.gov\.cn(.*?)\/(A.*?\.htm)/);
                         if (matchArray && matchArray.length > 2) {
                             const path = matchArray[1];
                             const htmlName = matchArray[2];
@@ -41,14 +41,14 @@ module.exports = async (ctx) => {
     const completeUrl = pathList.find((item) => item && item.path === path).completeUrl;
     const { data: channelResponse } = await got(completeUrl);
     const $1 = cheerio.load(channelResponse);
-    const items = $1('li.clearfix')
+    const items = $1('div#loadingInfoPage li')
         .toArray()
         .map((item) => {
             const c = $1(item);
             const a = c.find('a');
             const articleHref = a.attr('href');
             const title = a.text();
-            const date = parseDate(c.find('span.times').text());
+            const date = parseDate(c.find('.times').text());
             return {
                 link: articleHref,
                 pubDate: timezone(date),

package/lib/{routes → v2}/gov/chinatax/latest.js

@@ -1,4 +1,3 @@
-const url = require('url');
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 
@@ -17,19 +16,26 @@ module.exports = async (ctx) => {
             const a = item.find('a');
             return {
                 title: a.text(),
-                link: url.resolve(`http://www.chinatax.gov.cn`, a.attr('href')),
+                link: new URL(a.attr('href'), `http://www.chinatax.gov.cn`).toString(),
             };
         })
         .get();
-
     const items = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const res = await got({ method: 'get', url: item.link });
-                const content = cheerio.load(res.data);
-                item.pubDate = content('meta[name="PubDate"]').attr('content');
-                item.description = content('#fontzoom').html();
-                return item;
+                try {
+                    const res = await got({ method: 'get', url: item.link });
+                    const content = cheerio.load(res.data);
+                    item.pubDate = content('meta[name="PubDate"]').attr('content');
+                    item.description = content('#fontzoom').html();
+                    return item;
+                } catch (err) {
+                    if (err.name === 'HTTPError') {
+                        item.description = err.message;
+                        return item;
+                    }
+                    throw err;
+                }
             })
         )
     );

package/lib/v2/gov/maintainer.js

@@ -2,6 +2,7 @@ module.exports = {
     // ministry
     '/cac/:path+': ['drgnchan'],
     '/ccdi/:path+': ['bigfei'],
+    '/chinatax/latest': ['nczitzk', 'fuzy112'],
     '/cmse/xwzx/zhxw': ['nczitzk'],
     '/cmse/xwzx/yzjz': ['nczitzk'],
     '/cmse/gfgg': ['nczitzk'],
@@ -35,6 +36,7 @@ module.exports = {
     '/nmpa/:path+': ['TonyRL'],
     '/nopss/:path+': ['nczitzk'],
     '/nrta/news/:category?': ['yuxinliu-alex'],
+    '/nrta/dsj/:category?': ['nczitzk'],
     '/nsfc/news/:type?': ['Derekmini', 'nczitzk'],
     '/pbc/goutongjiaoliu': ['nczitzk'],
     '/pbc/gzlw': ['Fatpandac'],
@@ -71,6 +73,7 @@ module.exports = {
     '/shanghai/rsj/ksxm': ['Fatpandac'],
     '/shanghai/wsjkw/yqtb': ['zcf0508'],
     '/shanghai/yjj/:path+': ['nczitzk'],
+    '/shanghai/wgj/:page?': ['gideonsenku'],
     '/shenzhen/hrss/szksy/:caty/:page?': ['zlasd'],
     '/shenzhen/xxgk/zfxxgj/:caty': ['laoxua'],
     '/shenzhen/zzb/:caty/:page?': ['zlasd'],

package/lib/v2/gov/nifdc/index.js

@@ -36,29 +36,16 @@ module.exports = async (ctx) => {
             };
         });
 
-    const browser = await require('@/utils/puppeteer')();
-
     items = await Promise.all(
         items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 if (/^https:\/\/www\.nmpa\.gov\.cn\//.test(item.link)) {
-                    const page = await browser.newPage();
-                    await page.setRequestInterception(true);
-
-                    page.on('request', (request) => {
-                        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
-                    });
-
-                    await page.goto(item.link, {
-                        waitUntil: 'domcontentloaded',
-                    });
-                    await page.waitForSelector('.text');
-                    const html = await page.evaluate(() => document.documentElement.innerHTML);
-                    await page.close();
+                    const { data: html } = await got(item.link);
 
                     const content = cheerio.load(html);
 
                     item.description = content('.text').html();
+                    item.pubDate = timezone(parseDate($('meta[name="PubDate"]').attr('content')), +8);
                 } else {
                     const detailResponse = await got({
                         method: 'get',
@@ -76,8 +63,6 @@ module.exports = async (ctx) => {
         )
     );
 
-    await browser.close();
-
     ctx.state.data = {
         title: $('title').text().replace(/----/, ' - '),
         link: currentUrl,

package/lib/v2/gov/nmpa/generic.js

@@ -1,28 +1,18 @@
 const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const { finishArticleItem } = require('@/utils/wechat-mp');
 const config = require('@/config').value;
 const baseUrl = 'https://www.nmpa.gov.cn';
+const got = require('@/utils/got');
 
 module.exports = async (ctx) => {
     const path = ctx.params[0];
     const url = `${baseUrl}/${path.endsWith('/') ? path.slice(0, -1) : path}/index.html`;
-    const browser = await require('@/utils/puppeteer')({ stealth: true });
     const data = await ctx.cache.tryGet(
         url,
         async () => {
-            const page = await browser.newPage();
-            await page.setRequestInterception(true);
-            page.on('request', (request) => {
-                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
-            });
-            await page.goto(url, {
-                waitUntil: 'domcontentloaded',
-            });
-            await page.waitForSelector('.list');
-
-            const html = await page.evaluate(() => document.documentElement.innerHTML);
-            await page.close();
+            const { data: html } = await got(url);
             const $ = cheerio.load(html);
 
             return {
@@ -48,20 +38,10 @@ module.exports = async (ctx) => {
         data.items.map((item) => {
             if (/^https:\/\/www\.nmpa\.gov\.cn\//.test(item.link)) {
                 return ctx.cache.tryGet(item.link, async () => {
-                    const page = await browser.newPage();
-                    await page.setRequestInterception(true);
-                    page.on('request', (request) => {
-                        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
-                    });
-                    await page.goto(item.link, {
-                        waitUntil: 'domcontentloaded',
-                    });
-                    await page.waitForSelector('.text');
-
-                    const html = await page.evaluate(() => document.documentElement.innerHTML);
-                    await page.close();
+                    const { data: html } = await got(item.link);
                     const $ = cheerio.load(html);
                     item.description = $('.text').html();
+                    item.pubDate = timezone(parseDate($('meta[name="PubDate"]').attr('content')), +8);
                     return item;
                 });
             } else if (/^https:\/\/mp\.weixin\.qq\.com\//.test(item.link)) {
@@ -72,8 +52,6 @@ module.exports = async (ctx) => {
         })
     );
 
-    await browser.close();
-
     ctx.state.data = {
         title: data.title,
         description: data.description,

package/lib/v2/gov/nrta/dsj.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const asyncPool = require('tiny-async-pool');
+
+module.exports = async (ctx) => {
+    const { category = 'note' } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 15;
+
+    const rootUrl = 'https://dsj.nrta.gov.cn';
+    const currentUrl = new URL(`tims/site/views/applications.shanty?appName=${category}`, rootUrl).href;
+
+    const { data: response } = await got(currentUrl);
+
+    const $ = cheerio.load(response);
+
+    const items = $('img[src="/site/styles/default/images/icon_arrow_r.gif"]')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            item = $(item).next();
+
+            const pubDateMatches = item.text().match(/(\d+年\d+月)/);
+
+            return {
+                title: item.text(),
+                link: new URL(item.prop('href'), rootUrl).href,
+                pubDate: pubDateMatches ? parseDate(pubDateMatches[1], ['YYYY年MM月', 'YYYY年M月']) : undefined,
+            };
+        });
+
+    const results = [];
+
+    for await (const item of asyncPool(5, items, (item) =>
+        ctx.cache.tryGet(item.link, async () => {
+            const { data: detailResponse } = await got(item.link);
+
+            const content = cheerio.load(detailResponse);
+
+            content('table').last().remove();
+
+            item.description = content('td.newstext').html() || content('table').last().parent().parent().html();
+
+            return item;
+        })
+    )) {
+        results.push(item);
+    }
+
+    ctx.state.data = {
+        item: results,
+        title: `${$('title').text()}-${$('div.headbottom_menu_selected').text()}`,
+        link: currentUrl,
+        description: $('td').last().text(),
+        language: 'zh-cn',
+        image: $('img').first().prop('src'),
+        author: '国家广播电影电视总局电视剧管理司',
+    };
+};

package/lib/v2/gov/nsfc/index.js

@@ -1,74 +1,68 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 
-const types = {
-    jjyw: {
-        title: '基金要闻',
-        url: '/publish/portal0/tab440/',
-    },
-    tzgg: {
-        title: '通知公告',
-        url: '/publish/portal0/tab442/',
-    },
-    zzcg: {
-        title: '资助成果',
-        url: '/publish/portal0/tab448/',
-    },
-    kpkx: {
-        title: '科普快讯',
-        url: '/publish/portal0/tab446/',
-    },
-};
+const shortcuts = require('./shortcuts');
 
 module.exports = async (ctx) => {
-    const type = ctx.params.type ?? 'jjyw';
+    let thePath = ctx.path.replace(/^\/nsfc/, '');
+
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 30;
+
+    const shortcutMatches = thePath.match(/(\/news)?\/([\w-]+)/);
+
+    if (shortcutMatches) {
+        const shortcut = shortcutMatches[2];
+        if (shortcuts.hasOwnProperty(shortcut)) {
+            thePath = shortcuts[shortcut];
+        }
+    }
 
     const rootUrl = 'https://www.nsfc.gov.cn';
-    const currentUrl = `${rootUrl}${types[type].url}`;
+    const currentUrl = new URL((/\/more$/.test(thePath) ? `${thePath}.htm` : thePath) || 'publish/portal0/tab442/', rootUrl).href;
 
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
+    const { data: response } = await got(currentUrl, {
         https: {
             rejectUnauthorized: false,
         },
     });
 
-    const $ = cheerio.load(response.data);
+    const $ = cheerio.load(response);
 
-    let items = $('.fl a')
+    let items = $('span.fl a, ul.dp_lia li a')
+        .slice(0, limit)
         .toArray()
         .map((item) => {
             item = $(item);
 
-            const link = item.attr('href');
-
             return {
-                title: item.attr('title'),
-                link: `${/^http/.test(link) ? '' : rootUrl}${link}`,
-                pubDate: parseDate(item.parent().next().text()),
+                title: item.prop('title') ?? item.text(),
+                link: new URL(item.prop('href'), rootUrl).href,
+                guid: `nsfc-${item.prop('id')}`,
+                pubDate: parseDate(item.next().text().replace(/\[|\]/g, '', ['YYYY-MM-DD', 'YY-MM-DD'])),
             };
         });
 
     items = await Promise.all(
         items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                if (/nsfc\.gov\.cn/.test(item.link)) {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                        https: {
-                            rejectUnauthorized: false,
-                        },
-                    });
+                const { data: detailResponse } = await got(item.link, {
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
 
-                    const content = cheerio.load(detailResponse.data);
+                const content = cheerio.load(detailResponse);
 
-                    item.description = content('.content_xilan').html();
-                    item.pubDate = timezone(parseDate(content('meta[name="docdate"]').attr('content')), +8);
-                }
+                item.title = content('div.title_xilan').text();
+                item.description = content('#zoom').html();
+                item.author = content('meta[name="docauthor"]').prop('content');
+                item.category = [content('meta[name="channel"]').prop('content'), content('meta[name="docsource"]').prop('content')];
+                item.pubDate = parseDate(
+                    content('div.line_xilan')
+                        .text()
+                        .match(/日期 (\d{4}-\d{2}-\d{2})/)[1]
+                );
 
                 return item;
             })
@@ -76,8 +70,16 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: `国家自然科学基金委员会 - ${$('title').text()}`,
-        link: currentUrl,
         item: items,
+        title: `国家自然科学基金委员会 - ${$('#ess_essBREADCRUMB_lblBreadCrumb a.break')
+            .toArray()
+            .slice(1)
+            .map((a) => $(a).text())
+            .join(' - ')}`,
+        link: currentUrl,
+        description: $('meta[name="DESCRIPTION"]').prop('content'),
+        language: 'zh-cn',
+        subtitle: $('meta[name="KEYWORDS"]').prop('content'),
+        author: $('meta[name="AUTHOR"]').prop('content'),
     };
 };

package/lib/v2/gov/nsfc/shortcuts.js

@@ -0,0 +1,52 @@
+module.exports = {
+    // 基金要闻 - 基金要闻 https://www.nsfc.gov.cn/publish/portal0/tab440/
+    jjyw: '/publish/portal0/tab440/',
+
+    // 基金要闻 - 通知公告 https://www.nsfc.gov.cn/publish/portal0/tab442/
+    tzgg: '/publish/portal0/tab442/',
+
+    // 基金要闻 - 部门动态 https://www.nsfc.gov.cn/publish/portal0/tab445/
+    bmdt: '/publish/portal0/tab445/',
+
+    // 基金要闻 - 科普快讯 https://www.nsfc.gov.cn/publish/portal0/tab446/
+    kpkx: '/publish/portal0/tab446/',
+
+    // 基金要闻 - 资助成果 https://www.nsfc.gov.cn/publish/portal0/tab448/
+    zzcg: '/publish/portal0/tab448/',
+
+    // 政策法规 - 国家自然科学基金条例 https://www.nsfc.gov.cn/publish/portal0/tab471/
+    'zcfg-jjtl': '/publish/portal0/tab471/',
+
+    // 政策法规 - 国家自然科学基金发展规划 https://www.nsfc.gov.cn/publish/portal0/tab473/
+    'zcfg-fzgh': '/publish/portal0/tab473/',
+
+    // 政策法规 - 国家自然科学基金规章制度 https://www.nsfc.gov.cn/publish/portal0/tab475/
+    'zcfg-gzzd': '/publish/portal0/tab475/',
+
+    // 政策法规 - 国家科学技术相关法律法规 https://www.nsfc.gov.cn/publish/portal0/tab609/
+    'zcfg-flfg': '/publish/portal0/tab609/',
+
+    // 管理科学部 - 工作动态 https://www.nsfc.gov.cn/publish/portal0/tab1210/
+    'glkxb-gzdt': '/publish/portal0/tab1210/',
+
+    // 管理科学部 - 通知公告 https://www.nsfc.gov.cn/publish/portal0/tab1212/
+    'glkxb-tzgg': '/publish/portal0/tab1212/',
+
+    // 管理科学部 - 资助成果 https://www.nsfc.gov.cn/publish/portal0/tab1213/
+    'glkxb-zzcg': '/publish/portal0/tab1213/',
+
+    // 国际合作局 - 项目指南 https://www.nsfc.gov.cn/publish/portal0/tab1356/
+    'gjhzj-xmzn': '/publish/portal0/tab1356/',
+
+    // 国际合作局 - 初审结果 https://www.nsfc.gov.cn/publish/portal0/tab1361/
+    'gjhzj-csjg': '/publish/portal0/tab1361/',
+
+    // 国际合作局 - 批准通知 https://www.nsfc.gov.cn/publish/portal0/tab1362/
+    'gjhzj-pztz': '/publish/portal0/tab1362/',
+
+    // 国际合作局 - 进程简表 https://www.nsfc.gov.cn/publish/portal0/tab1366/
+    'gjhzj-jcjb': '/publish/portal0/tab1366/',
+
+    // 国际合作局 - 信息公开 https://www.nsfc.gov.cn/publish/portal0/tab1386/
+    'gjhzj-xxgk': '/publish/portal0/tab1386/',
+};