rsshub 1.0.0-master.fc5a2f6 → 1.0.0-master.fc648b5

package/lib/v2/elsevier/issue.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const journal = ctx.params.journal;
+    const issue = 'Volume ' + ctx.params.issue.replace('-', ' Issue ');
+    const host = 'https://www.sciencedirect.com';
+    const issueUrl = `${host}/journal/${journal}/vol/${ctx.params.issue.replace('-', '/issue/')}`;
+
+    const response = await got(issueUrl, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+    const jrnlName = $('.anchor.js-title-link').text();
+    const list = $('.js-article')
+        .map((_, item) => {
+            const title = $(item).find('.js-article-title').text();
+            const authors = $(item).find('.js-article__item__authors').text();
+            const link = $(item).find('.article-content-title').attr('href');
+            const id = $(item).find('.article-content-title').attr('id');
+            return {
+                title,
+                link,
+                id,
+                authors,
+                issue,
+            };
+        })
+        .get();
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response2 = await got(`${host}/science/article/pii/${item.id}`, {
+                    cookieJar,
+                });
+
+                const $2 = cheerio.load(response2.data);
+                $2('.section-title').remove();
+                item.doi = $2('.doi').attr('href').replace('https://doi.org/', '');
+                item.abstract = $2('.abstract.author').text();
+                item.description = renderDesc(item);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${jrnlName} - ${issue}`,
+        link: issueUrl,
+        item: items,
+    };
+};

package/lib/v2/elsevier/journal.js

@@ -0,0 +1,72 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const journal = ctx.params.journal;
+    const host = 'https://www.sciencedirect.com';
+    const jrnlUrl = `${host}/journal/${journal}`;
+
+    const response = await got(jrnlUrl, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+    const jrnlName = $('.anchor.js-title-link').text();
+    const issueUrl = `${host}${$('.anchor.js-volume').attr('href')}`;
+    let issue = '';
+    if (issueUrl.match('suppl') !== null) {
+        issue = 'Volume ' + issueUrl.match('vol/(.*)/suppl')[1];
+    } else if (issueUrl.match('issue') !== null) {
+        issue = 'Volume ' + issueUrl.match('vol/(.*)/issue')[1] + ' Issue ' + issueUrl.match('/issue/(.*)')[1];
+    }
+
+    const response2 = await got(issueUrl, {
+        cookieJar,
+    });
+    const $2 = cheerio.load(response2.data);
+    const list = $2('.js-article')
+        .map((_, item) => {
+            const title = $2(item).find('.js-article-title').text();
+            const authors = $2(item).find('.js-article__item__authors').text();
+            const link = $2(item).find('.article-content-title').attr('href');
+            const id = $2(item).find('.article-content-title').attr('id');
+            return {
+                title,
+                link,
+                id,
+                authors,
+                issue,
+            };
+        })
+        .get();
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response3 = await got(`${host}/science/article/pii/${item.id}`, {
+                    cookieJar,
+                });
+                const $3 = cheerio.load(response3.data);
+                $3('.section-title').remove();
+                item.doi = $3('.doi').attr('href').replace('https://doi.org/', '');
+                item.abstract = $3('.abstract.author').text();
+                item.description = renderDesc(item);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: jrnlName,
+        link: jrnlUrl,
+        item: items,
+    };
+};

package/lib/v2/elsevier/maintainer.js

@@ -1,4 +1,4 @@
 module.exports = {
-    '/:journal/latest': ['Derekmini', 'sunwolf-swb'],
-    '/:journal/vol/:id': ['Derekmini', 'sunwolf-swb'],
+    '/:journal': ['Derekmini', 'sunwolf-swb'],
+    '/:journal/:issue': ['Derekmini', 'sunwolf-swb'],
 };

package/lib/v2/elsevier/radar.js

@@ -3,16 +3,16 @@ module.exports = {
         _name: 'Elsevier',
         www: [
             {
-                title: 'latest',
+                title: 'Journal',
                 docs: 'https://docs.rsshub.app/journal.html#Elsevier',
                 source: '/journal/:journal/*',
-                target: '/elsevier/:journal/latest',
+                target: '/elsevier/:journal',
             },
             {
-                title: 'volume',
+                title: 'Issue',
                 docs: 'https://docs.rsshub.app/journal.html#Elsevier',
-                source: '/journal/:journal/vol/:id',
-                target: '/elsevier/:journal/vol/:id',
+                source: '/journal/:journal/vol/:issue',
+                target: '/elsevier/:journal/:issue',
             },
         ],
     },

package/lib/v2/elsevier/router.js

@@ -1,4 +1,6 @@
 module.exports = function (router) {
-    router.get('/:journal/latest', require('./latest'));
-    router.get('/:journal/vol/:id', require('./volume'));
+    router.get('/:journal/latest', require('./journal')); // deprecated
+    router.get('/:journal/vol/:issue', require('./issue')); // deprecated
+    router.get('/:journal', require('./journal'));
+    router.get('/:journal/:issue', require('./issue'));
 };

package/lib/v2/elsevier/templates/description.art

@@ -1,11 +1,11 @@
 <p>
-    <span><big>{{ item.title }}</big></span><br />
+    <span><big>{{ item.title }}</big></span><br>
 </p>
 <p>
-    <span><small><i>{{ item.authors }}</i></small></span><br />
-    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br />
-    <span><small><i>{{ item.volume }}</i></small></span><br />
+    <span><small><i>{{ item.authors }}</i></small></span><br>
+    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br>
+    <span><small><i>{{ item.issue }}</i></small></span><br>
 </p>
 <p>
-    <span>{{ item.abstract }}</span><br />    
+    <span>{{ item.abstract }}</span><br>
 </p>

package/lib/v2/gov/beijing/jw/tzgg.js

@@ -0,0 +1,58 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'http://jw.beijing.gov.cn';
+    const currentUrl = `${rootUrl}/tzgg`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.col-md a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href');
+
+            return {
+                title: item.text(),
+                link: /^http/.test(link) ? link : `${rootUrl}${link.replace(/^\./, '/tzgg')}`,
+                pubDate: parseDate(item.parent().find('span').text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                const pubDate = content('meta[name="PubDate"]').attr('content');
+
+                item.author = content('meta[name="ContentSource"]').attr('content');
+                item.pubDate = pubDate ? timezone(parseDate(content('meta[name="PubDate"]').attr('content')), +8) : item.pubDate;
+                item.description = content('.TRS_UEDITOR').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '北京市教育委员会 - 通知公告',
+        link: currentUrl,
+        item: items,
+        description: $('meta[name="ColumnDescription"]').attr('content'),
+    };
+};

package/lib/v2/gov/customs/list.js

@@ -23,33 +23,34 @@ module.exports = async (ctx) => {
             link = `${host}/customs/302249/zfxxgk/2799825/2799883/index.html`;
             break;
     }
-    const response = await ctx.cache.tryGet(
+
+    const browser = await require('@/utils/puppeteer')();
+
+    const list = await ctx.cache.tryGet(
         link,
         async () => {
-            const response = await puppeteerGet(link);
-            return response;
+            const response = await puppeteerGet(link, browser);
+            const $ = cheerio.load(response);
+            const list = $('[class^="conList_ul"] li')
+                .toArray()
+                .map((item) => {
+                    item = $(item);
+                    return {
+                        title: item.find('a').attr('title'),
+                        link: new URL(item.find('a').attr('href'), host).href,
+                        date: parseDate(item.find('span').text()),
+                    };
+                });
+            return list;
         },
         config.cache.routeExpire,
         false
     );
 
-    const $ = cheerio.load(response);
-
-    const list = $('[class^="conList_ul"] li')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-            return {
-                title: item.find('a').attr('title'),
-                link: new URL(item.find('a').attr('href'), host).href,
-                date: parseDate(item.find('span').text()),
-            };
-        });
-
     const out = await Promise.all(
         list.map((info) =>
             ctx.cache.tryGet(info.link, async () => {
-                const response = await puppeteerGet(info.link);
+                const response = await puppeteerGet(info.link, browser);
                 const $ = cheerio.load(response);
                 let date;
 
@@ -68,6 +69,9 @@ module.exports = async (ctx) => {
             })
         )
     );
+
+    browser.close();
+
     ctx.state.data = {
         title: `中国海关-${channelName}`,
         link,

package/lib/v2/gov/customs/utils.js

@@ -1,13 +1,17 @@
 const host = 'http://www.customs.gov.cn';
 
-const puppeteerGet = async (url) => {
-    const browser = await require('@/utils/puppeteer')({ stealth: true });
+const puppeteerGet = async (url, browser) => {
     const page = await browser.newPage();
-    page.setExtraHTTPHeaders({ referer: host });
-    await page.goto(url);
+    await page.setExtraHTTPHeaders({ referer: host });
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
     await page.waitForSelector('.pubCon');
     const html = await page.evaluate(() => document.documentElement.innerHTML);
-    browser.close();
     return html;
 };
 

package/lib/v2/gov/maintainer.js

@@ -11,6 +11,7 @@ module.exports = {
     '/miit/wjgs': ['Yoge-Code'],
     '/miit/zcjd': ['Yoge-Code'],
     '/moe/:type': ['Crawler995'],
+    '/nmpa/:path+': ['TonyRL'],
     '/nrta/news/:category?': ['yuxinliu-alex'],
     '/pbc/goutongjiaoliu': ['nczitzk'],
     '/pbc/gzlw': ['Fatpandac'],
@@ -18,6 +19,7 @@ module.exports = {
     '/pbc/zcyj': ['Fatpandac'],
     // province
     '/anhui/kjt/:path?': ['nczitzk'],
+    '/beijing/jw/tzgg': ['nczitzk'],
     '/beijing/kw/:channel': ['Fatpandac'],
     '/hebei/czt/xwdt/:category?': ['nczitzk'],
     '/shenzhen/hrss/szksy/:caty/:page?': ['zlasd'],
@@ -28,4 +30,5 @@ module.exports = {
     '/guangdong/tqyb/sncsyjxh': ['Fatpandac'],
     '/huizhou/zwgk/:category?': ['Fatpandac'],
     '/sichuan/deyang/govpulicinfo/:countyName': ['zytomorrow'],
+    '/xuzhou/hrss/:category?': ['nczitzk'],
 };

package/lib/v2/gov/nmpa/generic.js

@@ -0,0 +1,76 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const baseUrl = 'https://www.nmpa.gov.cn';
+
+module.exports = async (ctx) => {
+    const path = ctx.params[0];
+    const url = `${baseUrl}/${path.endsWith('/') ? path.slice(0, -1) : path}/index.html`;
+    const browser = await require('@/utils/puppeteer')();
+    const data = await ctx.cache.tryGet(
+        url,
+        async () => {
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
+            await page.goto(url, {
+                waitUntil: 'domcontentloaded',
+            });
+            await page.waitForSelector('.list');
+
+            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            await page.close();
+            const $ = cheerio.load(html);
+
+            return {
+                title: $('head title').text(),
+                description: $('meta[name=ColumnDescription]').attr('content'),
+                items: $('.list ul li')
+                    .toArray()
+                    .map((item) => {
+                        item = $(item);
+                        return {
+                            title: item.find('a').text().trim(),
+                            link: new URL(item.find('a').attr('href'), baseUrl).href,
+                            pubDate: parseDate(item.find('span').text(), 'YYYY-MM-DD'),
+                        };
+                    }),
+            };
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const items = await Promise.all(
+        data.items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await page.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
+                });
+                await page.waitForSelector('.text');
+
+                const html = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
+                const $ = cheerio.load(html);
+                item.description = $('.text').html();
+                return item;
+            })
+        )
+    );
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: data.title,
+        description: data.description,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/gov/pbc/goutongjiaoliu.js

@@ -5,22 +5,25 @@ const timezone = require('@/utils/timezone');
 module.exports = async (ctx) => {
     const link = 'http://www.pbc.gov.cn/goutongjiaoliu/113456/113469/index.html';
 
-    const browser = await require('@/utils/puppeteer')();
+    const browser = await require('@/utils/puppeteer')({ stealth: true });
     const page = await browser.newPage();
-    await page.goto(link);
-    const html = await page.evaluate(
-        () =>
-            // eslint-disable-next-line
-            document.querySelector('body').innerHTML
-    );
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(link, {
+        waitUntil: 'domcontentloaded',
+    });
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+
     const $ = cheerio.load(html);
     const list = $('font.newslist_style')
         .map((_, item) => {
             item = $(item);
             const a = item.find('a[title]');
             return {
-                title: a.text(),
-                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn'),
+                title: a.attr('title'),
+                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn').href,
             };
         })
         .get();
@@ -29,12 +32,14 @@ module.exports = async (ctx) => {
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailPage = await browser.newPage();
-                await detailPage.goto(item.link);
-                const detailHtml = await detailPage.evaluate(
-                    () =>
-                        // eslint-disable-next-line
-                        document.querySelector('body').innerHTML
-                );
+                await detailPage.setRequestInterception(true);
+                detailPage.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await detailPage.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
+                });
+                const detailHtml = await detailPage.evaluate(() => document.documentElement.innerHTML);
                 const content = cheerio.load(detailHtml);
                 item.description = content('#zoom').html();
                 item.pubDate = timezone(parseDate(content('.hui12').eq(5).text()), +8);

package/lib/v2/gov/pbc/tradeAnnouncement.js

@@ -7,20 +7,22 @@ module.exports = async (ctx) => {
 
     const browser = await require('@/utils/puppeteer')();
     const page = await browser.newPage();
-    await page.goto(link);
-    const html = await page.evaluate(
-        () =>
-            // eslint-disable-next-line
-            document.querySelector('body').innerHTML
-    );
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(link, {
+        waitUntil: 'domcontentloaded',
+    });
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
     const $ = cheerio.load(html);
     const list = $('font.newslist_style')
         .map((_, item) => {
             item = $(item);
             const a = item.find('a[title]');
             return {
-                title: a.text(),
-                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn'),
+                title: a.attr('title'),
+                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn').href,
             };
         })
         .get();
@@ -29,12 +31,14 @@ module.exports = async (ctx) => {
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailPage = await browser.newPage();
-                await detailPage.goto(item.link);
-                const detailHtml = await detailPage.evaluate(
-                    () =>
-                        // eslint-disable-next-line
-                        document.querySelector('body').innerHTML
-                );
+                await detailPage.setRequestInterception(true);
+                detailPage.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await detailPage.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
+                });
+                const detailHtml = await detailPage.evaluate(() => document.documentElement.innerHTML);
                 const content = cheerio.load(detailHtml);
                 item.description = content('#zoom').html();
                 item.pubDate = timezone(parseDate(content('#shijian').text()), +8);