rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/lib/{routes → v2}/hk01/hot.js

@@ -1,4 +1,5 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const response = await got('https://web-data.api.hk01.com/v2/page/hot/');
@@ -13,7 +14,7 @@ module.exports = async (ctx) => {
             title: item.data.title,
             author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
             description: `<p>${item.data.description}</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`,
-            pubDate: new Date(item.data.lastModifyTime * 1000),
+            pubDate: parseDate(item.data.lastModifyTime * 1000),
             link: item.data.canonicalUrl,
         })),
     };

package/lib/v2/hk01/issue.js

@@ -0,0 +1,41 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const response = await got(`https://web-data.api.hk01.com/v2/page/issue/${id}`);
+    const data = response.data;
+    const list = data.issue.blocks[0].articles;
+
+    await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(`https://www.hk01.com/${item.data.mainCategory}/${item.data.articleId}`, async () => {
+                const detailResponse = await got(item.data.canonicalUrl);
+                const $ = cheerio.load(detailResponse.data);
+
+                $('div.lazyload-placeholder').remove();
+                $('article > div.view-tracker').remove();
+                $('article > div.flex.flex-col').remove();
+
+                item.data.description = $('article').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `香港01 - ${data.issue.title}`,
+        description: data.meta.metaDesc,
+        link: data.issue.publishUrl,
+        item: list.map((item) => ({
+            title: item.data.title,
+            author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
+            description: `<img style="width: 100%" src="${item.data.mainImage.cdnUrl}" /><p>${item.data.description}</p>`,
+            pubDate: parseDate(item.data.lastModifyTime * 1000),
+            link: item.data.canonicalUrl,
+        })),
+    };
+};

package/lib/v2/hk01/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/channel/:id': ['hoilc', 'Fatpandac'],
+    '/hot': ['hoilc', 'Fatpandac'],
+    '/issue/:id': ['hoilc', 'Fatpandac'],
+    '/tag/:id': ['hoilc', 'Fatpandac'],
+    '/zone/:id': ['hoilc', 'Fatpandac'],
+};

package/lib/v2/hk01/radar.js

@@ -0,0 +1,37 @@
+module.exports = {
+    'hk01.com': {
+        _name: '香港01',
+        www: [
+            {
+                title: '最 Hit',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: ['/hot', '/'],
+                target: '/hk01/hot',
+            },
+            {
+                title: 'zone',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: '/zone/:id/:title',
+                target: '/hk01/zone/:id',
+            },
+            {
+                title: 'channel',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: '/channel/:id/:title',
+                target: '/hk01/channel/:id',
+            },
+            {
+                title: 'issue',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: '/issue/:id/:title',
+                target: '/hk01/issue/:id',
+            },
+            {
+                title: 'tag',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: '/tag/:id/:title',
+                target: '/hk01/tag/:id',
+            },
+        ],
+    },
+};

package/lib/v2/hk01/router.js

@@ -0,0 +1,7 @@
+module.exports = function (router) {
+    router.get('/channel/:id', require('./channel'));
+    router.get('/hot', require('./hot'));
+    router.get('/issue/:id', require('./issue'));
+    router.get('/tag/:id', require('./tag'));
+    router.get('/zone/:id', require('./zone'));
+};

package/lib/{routes → v2}/hk01/tag.js

@@ -1,4 +1,5 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
@@ -15,7 +16,7 @@ module.exports = async (ctx) => {
             title: item.data.title,
             author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
             description: `<p>${item.data.description}</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`,
-            pubDate: new Date(item.data.lastModifyTime * 1000),
+            pubDate: parseDate(item.data.lastModifyTime * 1000),
             link: item.data.canonicalUrl,
         })),
     };

package/lib/{routes → v2}/hk01/zone.js

@@ -1,4 +1,5 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
@@ -19,12 +20,12 @@ module.exports = async (ctx) => {
                 case 1:
                     author = item.data.authors && item.data.authors.map((e) => e.publishName).join(', ');
                     description = `<p>${item.data.description}</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`;
-                    pubDate = new Date(item.data.lastModifyTime * 1000);
+                    pubDate = parseDate(item.data.lastModifyTime * 1000);
                     break;
                 case 2:
                     author = item.data.zonePublishName;
                     description = `${item.data.teaser.map((e) => '<p>' + e + '</p>').join('')}<img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`;
-                    pubDate = new Date();
+                    pubDate = parseDate(item.data.lastModifyTime * 1000);
                     break;
                 default:
                     break;

package/lib/v2/hotukdeals/index.js

@@ -24,7 +24,7 @@ module.exports = async (ctx) => {
                 item = $(item);
                 return {
                     title: item.find('.cept-tt').text(),
-                    description: `${item.find('.cept-thread-image-link').html()}<br>${item.find('.cept-vote-temp').html()}<br>${item.find('.overflow--fade').html()}<br>${item.find('.cept-description-container').html()}`,
+                    description: `${item.find('.thread-listImgCell').html()}<br>${item.find('.cept-vote-temp').html()}<br>${item.find('.overflow--fade').html()}<br>${item.find('.threadGrid-body .userHtml').html()}`,
                     link: item.find('.cept-tt').attr('href'),
                 };
             })

package/lib/{routes → v2}/huxiu/article.js

@@ -14,7 +14,7 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('.article-item--normal > a, .article-item--large > a')
+    const list = $('.article-item-wrap > a')
         .slice(0, 10)
         .get()
         .map((e) => $(e).attr('href'));

package/lib/{routes → v2}/huxiu/author.js

@@ -4,7 +4,8 @@ const utils = require('./utils');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const link = `https://www.huxiu.com/member/${id}.html`;
+    const link = `https://www.huxiu.com/member/${id}/article.html`;
+
     const response = await got({
         method: 'get',
         url: link,
@@ -14,12 +15,23 @@ module.exports = async (ctx) => {
     });
 
     const $ = cheerio.load(response.data);
-    const author = $('.user-name').text().trim();
+    const author = $('.author-name').text().trim();
+
+    const articleList = await got({
+        method: 'post',
+        url: 'https://account-api.huxiu.com/web/article/articleList',
+        headers: {
+            Referer: 'https://www.huxiu.com/',
+        },
+        form: {
+            platform: 'www',
+            uid: id,
+            type: 0,
+            page: 1,
+        },
+    }).json();
 
-    const list = $('.message-box > .mod-art > a')
-        .slice(0, 10)
-        .get()
-        .map((e) => $(e).attr('href'));
+    const list = articleList.data.datalist.slice(0, 10).map((item) => item.url);
 
     const items = await utils.ProcessFeed(list, ctx.cache);
 

package/lib/{routes → v2}/huxiu/collection.js

@@ -16,8 +16,7 @@ module.exports = async (ctx) => {
     const data = response.data;
     const $ = cheerio.load(data);
 
-    const list = $('.collection__article-list__item > a')
-        .filter('.article-item')
+    const list = $('.collection-lirt-item > a')
         .slice(0, 5)
         .get()
         .map((e) => $(e).attr('href'));

package/lib/v2/huxiu/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/article': ['HenryQW'],
+    '/author/:id': ['HenryQW'],
+    '/collection/:id': ['AlexdanerZe'],
+    '/tag/:id': ['xyqfer', 'HenryQW'],
+    '/search/:keyword': ['xyqfer', 'HenryQW'],
+};

package/lib/v2/huxiu/radar.js

@@ -0,0 +1,35 @@
+module.exports = {
+    'huxiu.com': {
+        _name: '虎嗅',
+        '.': [
+            {
+                title: '首页资讯',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+                source: ['/article', '/'],
+                target: '/huxiu/article',
+            },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+                source: ['/tags/:id'],
+                target: (params) => `/huxiu/tag/${params.id.replace('.html', '')}`,
+            },
+            {
+                title: '搜索',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+            },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+                source: ['/member/:id/*', '/'],
+                target: '/huxiu/author/:id',
+            },
+            {
+                title: '文集',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+                source: ['/collection/:id', '/'],
+                target: (params) => `/huxiu/collection/${params.id.replace('.html', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/huxiu/router.js

@@ -0,0 +1,7 @@
+module.exports = (router) => {
+    router.get('/article', require('./article'));
+    router.get('/author/:id', require('./author'));
+    router.get('/collection/:id', require('./collection'));
+    router.get('/tag/:id', require('./tag'));
+    router.get('/search/:keyword', require('./search'));
+};

package/lib/{routes → v2}/huxiu/utils.js

@@ -1,17 +1,16 @@
 const cheerio = require('cheerio');
 const date = require('@/utils/date');
 const got = require('@/utils/got');
-const url = require('url');
 
 const ProcessFeed = async (list, cache) => {
     const host = 'https://www.huxiu.com';
 
     const items = await Promise.all(
         list.map(async (e) => {
-            const link = url.resolve(host, e);
+            const link = new URL(e, host).href;
 
             const single = await cache.tryGet(link, async () => {
-                const response = await got.get(link);
+                const response = await got(link);
 
                 const $ = cheerio.load(response.data);
 
@@ -58,7 +57,7 @@ const ProcessFeed = async (list, cache) => {
                 };
             });
 
-            return Promise.resolve(single);
+            return single;
         })
     );
 

package/lib/v2/ieee/journal.js

@@ -0,0 +1,78 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const punumber = ctx.params.journal;
+    const sortType = ctx.params.sortType ?? 'vol-only-seq';
+    const host = 'https://ieeexplore.ieee.org';
+    const jrnlUrl = `${host}/xpl/mostRecentIssue.jsp?punumber=${punumber}`;
+
+    const response = await got(`${host}/rest/publication/home/metadata?pubid=${punumber}`, {
+        cookieJar,
+    }).json();
+    const volume = response.currentIssue.volume;
+    const isnumber = response.currentIssue.issueNumber;
+    const jrnlName = response.displayTitle;
+
+    const response2 = await got
+        .post(`${host}/rest/search/pub/${punumber}/issue/${isnumber}/toc`, {
+            cookieJar,
+            json: {
+                punumber,
+                isnumber,
+                sortType,
+                rowsPerPage: '100',
+            },
+        })
+        .json();
+    let list = response2.records.map((item) => {
+        const $2 = cheerio.load(item.articleTitle);
+        const title = $2.text();
+        const link = item.htmlLink;
+        const doi = item.doi;
+        let authors = 'Do not have author';
+        if (item.hasOwnProperty('authors')) {
+            authors = item.authors.map((itemAuth) => itemAuth.preferredName).join('; ');
+        }
+        let abstract = '';
+        item.hasOwnProperty('abstract') ? (abstract = item.abstract) : (abstract = '');
+        return {
+            title,
+            link,
+            authors,
+            doi,
+            volume,
+            abstract,
+        };
+    });
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    list = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (item.abstract !== '') {
+                    const response3 = await got(`${host}${item.link}`);
+                    const { abstract } = JSON.parse(response3.body.match(/metadata=(.*);/)[1]);
+                    const $3 = cheerio.load(abstract);
+                    item.abstract = $3.text();
+                    item.description = renderDesc(item);
+                }
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: jrnlName,
+        link: jrnlUrl,
+        item: list,
+    };
+};

package/lib/v2/ieee/maintainer.js

@@ -1,4 +1,4 @@
 module.exports = {
-    '/:journal/latest/date/:sortType?': ['Derekmini', 'auto-bot-ty'],
-    '/:journal/latest/vol/:sortType?': ['Derekmini', 'auto-bot-ty'],
+    '/journal/:journal/:sortType?': ['Derekmini', 'auto-bot-ty'],
+    '/journal/:journal/recent/:sortType?': ['Derekmini', 'auto-bot-ty'],
 };

package/lib/v2/ieee/radar.js

@@ -3,16 +3,16 @@ module.exports = {
         _name: 'IEEE',
         www: [
             {
-                title: 'latestVolume',
+                title: 'Journal',
                 docs: 'https://docs.rsshub.app/journal.html#ieee-xplore',
                 source: '/*',
-                target: (params, url) => `/ieee/${new URL(url).searchParams.get('punumber')}/latest/vol`,
+                target: (params, url) => `/ieee/journal/${new URL(url).searchParams.get('punumber')}`,
             },
             {
-                title: 'latestDate',
+                title: 'Recent',
                 docs: 'https://docs.rsshub.app/journal.html#ieee-xplore',
                 source: '/*',
-                target: (params, url) => `/ieee/${new URL(url).searchParams.get('punumber')}/latest/date`,
+                target: (params, url) => `/ieee/journal/${new URL(url).searchParams.get('punumber')}/recent`,
             },
         ],
     },

package/lib/v2/ieee/{latestdate.js → recent.js}

@@ -1,21 +1,17 @@
 const got = require('@/utils/got');
-const config = require('@/config').value;
+const cheerio = require('cheerio');
 const path = require('path');
 const { art } = require('@/utils/render');
 
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
 module.exports = async (ctx) => {
-    const host = 'https://ieeexplore.ieee.org';
     const punumber = ctx.params.journal;
     const sortType = ctx.params.sortType ?? 'vol-only-seq';
-    const hostUrl = host.concat('/xpl/mostRecentIssue.jsp?punumber=', punumber);
-    const headers = {
-        Accept: 'application/json, text/plain, */*',
-        'Accept-Encoding': 'gzip, deflate, br',
-        'Accept-Language': 'zh-CN,zh;q=0.9',
-        Host: host.replace('https://', ''),
-        'User-Agent': config.ua,
-        Referer: hostUrl,
-    };
+    const host = 'https://ieeexplore.ieee.org';
+    const jrnlUrl = `${host}/xpl/mostRecentIssue.jsp?punumber=${punumber}`;
+
     const date = new Date();
     const year = date.getFullYear();
     const month = date.getMonth() + 1;
@@ -32,29 +28,28 @@ module.exports = async (ctx) => {
         endYM = year + '' + month;
     }
 
-    const resp = await got({
-        method: 'get',
-        url: host.concat('/rest/publication/home/metadata?pubid=', punumber),
-        headers,
+    const response = await got(`${host}/rest/publication/home/metadata?pubid=${punumber}`, {
+        cookieJar,
     }).json();
-    const volume = resp.currentIssue.volume;
-    const isnumber = resp.currentIssue.issueNumber;
-    const jrnlName = resp.displayTitle;
+    const volume = response.currentIssue.volume;
+    const isnumber = response.currentIssue.issueNumber;
+    const jrnlName = response.displayTitle;
 
-    const response = await got({
-        method: 'post',
-        url: host.concat('/rest/search/pub/', punumber, '/issue/', isnumber, '/toc'),
-        headers,
-        json: {
-            punumber,
-            isnumber,
-            sortType,
-            rowsPerPage: '100',
-            ranges: [strYM + `01_` + endYM + `31_Search Latest Date`],
-        },
-    }).json();
-    const list = response.records.map((item) => {
-        const title = item.articleTitle;
+    const response2 = await got
+        .post(`${host}/rest/search/pub/${punumber}/issue/${isnumber}/toc`, {
+            cookieJar,
+            json: {
+                punumber,
+                isnumber,
+                sortType,
+                rowsPerPage: '100',
+                ranges: [strYM + `01_` + endYM + `31_Search Latest Date`],
+            },
+        })
+        .json();
+    let list = response2.records.map((item) => {
+        const $2 = cheerio.load(item.articleTitle);
+        const title = $2.text();
         const link = item.htmlLink;
         const doi = item.doi;
         let authors = 'Do not have author';
@@ -77,17 +72,14 @@ module.exports = async (ctx) => {
         art(path.join(__dirname, 'templates/description.art'), {
             item,
         });
-    await Promise.all(
+    list = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 if (item.abstract !== '') {
-                    const itemResponse = await got({
-                        method: 'get',
-                        url: host.concat(item.link),
-                    });
-                    const content = itemResponse.body.match(/metadata=(.*);/)[1];
-                    const { abstract } = JSON.parse(content);
-                    item.abstract = abstract;
+                    const response3 = await got(`${host}${item.link}`);
+                    const { abstract } = JSON.parse(response3.body.match(/metadata=(.*);/)[1]);
+                    const $3 = cheerio.load(abstract);
+                    item.abstract = $3.text();
                     item.description = renderDesc(item);
                 }
                 return item;
@@ -96,8 +88,8 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: jrnlName.concat(' - Latest'),
-        link: hostUrl,
+        title: `${jrnlName} - Recent`,
+        link: jrnlUrl,
         item: list,
     };
 };

package/lib/v2/ieee/router.js

@@ -1,4 +1,6 @@
 module.exports = function (router) {
-    router.get('/:journal/latest/date/:sortType?', require('./latestdate'));
-    router.get('/:journal/latest/vol/:sortType?', require('./latestvol'));
+    router.get('/:journal/latest/vol/:sortType?', require('./journal')); // deprecated
+    router.get('/:journal/latest/date/:sortType?', require('./recent')); // deprecated
+    router.get('/journal/:journal/:sortType?', require('./journal'));
+    router.get('/journal/:journal/recent/:sortType?', require('./recent'));
 };

package/lib/v2/ieee/templates/description.art

@@ -1,11 +1,11 @@
 <p>
-    <span><big>{{ item.title }}</big></span><br />
+    <span><big>{{ item.title }}</big></span><br>
 </p>
 <p>
-    <span><small><i>{{ item.authors }}</i></small></span><br />
-    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br />
-    <span><small><i>Volume: {{ item.volume }}</i></small></span><br />
+    <span><small><i>{{ item.authors }}</i></small></span><br>
+    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br>
+    <span><small><i>Volume {{ item.volume }}</i></small></span><br>
 </p>
 <p>
-    <span>{{ item.abstract }}</span><br />    
+    <span>{{ item.abstract }}</span><br>
 </p>

package/lib/v2/ielts/index.js

@@ -3,37 +3,43 @@ const got = require('@/utils/got');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const targetUrl = 'https://ielts.neea.cn/allnews?locale=zh_CN';
+const config = require('@/config').value;
 
 module.exports = async (ctx) => {
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    await page.evaluateOnNewDocument(() => {
-        const proto = navigator.__proto__;
-        delete proto.webdriver;
-    });
+    const html = await ctx.cache.tryGet(
+        targetUrl,
+        async () => {
+            const browser = await require('@/utils/puppeteer')({ stealth: true });
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
+            await page.goto(targetUrl, {
+                waitUntil: 'domcontentloaded',
+            });
+            await page.waitForSelector('div.container');
 
-    await page.goto(targetUrl, {
-        waitUntil: 'networkidle0',
-    });
-    const list = await page.evaluate(() =>
-        window
-            .$('#newsListUl li')
-            .get()
-            .map((elem) => {
-                const $elem = window.$(elem);
-                return {
-                    title: $elem.find('a').text(),
-                    link: $elem.find('a').attr('href'),
-                    pubDate: $elem.find('span').eq(-1).text().replace(/[[\]]/g, '').trim(),
-                };
-            })
+            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            browser.close();
+            return html;
+        },
+        config.cache.routeExpire,
+        false
     );
-    browser.close();
 
-    list.sort((a, b) => parseDate(b.pubDate) - parseDate(a.pubDate));
-    list.forEach((item) => {
-        item.pubDate = timezone(parseDate(item.pubDate), +8);
-    });
+    const $ = cheerio.load(html);
+
+    const list = $('#newsListUl li')
+        .get()
+        .map((elem) => {
+            const $elem = $(elem);
+            return {
+                title: $elem.find('a').text(),
+                link: $elem.find('a').attr('href'),
+                pubDate: timezone(parseDate($elem.find('span').eq(-1).text().replace(/[[\]]/g, '').trim(), +8)),
+            };
+        });
 
     const items = await Promise.all(
         list.map((item) =>