rsshub 1.0.0-master.fc5a2f6 → 1.0.0-master.fc648b5

package/lib/v2/heu/yjsy/list.js

@@ -39,7 +39,7 @@ module.exports = async (ctx) => {
         })
         .get();
 
-    await Promise.all(
+    const items = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 if (item.link.indexOf('page.htm') !== -1) {
@@ -60,6 +60,6 @@ module.exports = async (ctx) => {
     ctx.state.data = {
         title: '研究生院-' + bigTitle,
         link: rootUrl.concat('/', id, '/list.htm'),
-        item: list,
+        item: items,
     };
 };

package/lib/v2/hk01/channel.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const response = await got(`https://web-data.api.hk01.com/v2/page/category/${id}`);
+    const data = response.data;
+    const list = data.sections[0].items;
+
+    await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(`https://www.hk01.com/${item.data.mainCategory}/${item.data.articleId}`, async () => {
+                const detailResponse = await got(item.data.canonicalUrl);
+                const $ = cheerio.load(detailResponse.data);
+
+                $('article > div.view-tracker').remove();
+                $('article > div.flex.flex-col').remove();
+
+                item.data.description = $('article').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `香港01 - ${data.category.publishName}`,
+        description: data.meta.metaDesc,
+        link: data.category.publishUrl,
+        item: list.map((item) => ({
+            title: item.data.title,
+            author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
+            description: `<p>${item.data.description}</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`,
+            pubDate: parseDate(item.data.lastModifyTime * 1000),
+            link: item.data.canonicalUrl,
+        })),
+    };
+};

package/lib/{routes → v2}/hk01/hot.js

@@ -1,4 +1,5 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const response = await got('https://web-data.api.hk01.com/v2/page/hot/');
@@ -13,7 +14,7 @@ module.exports = async (ctx) => {
             title: item.data.title,
             author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
             description: `<p>${item.data.description}</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`,
-            pubDate: new Date(item.data.lastModifyTime * 1000),
+            pubDate: parseDate(item.data.lastModifyTime * 1000),
             link: item.data.canonicalUrl,
         })),
     };

package/lib/v2/hk01/issue.js

@@ -0,0 +1,41 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const response = await got(`https://web-data.api.hk01.com/v2/page/issue/${id}`);
+    const data = response.data;
+    const list = data.issue.blocks[0].articles;
+
+    await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(`https://www.hk01.com/${item.data.mainCategory}/${item.data.articleId}`, async () => {
+                const detailResponse = await got(item.data.canonicalUrl);
+                const $ = cheerio.load(detailResponse.data);
+
+                $('div.lazyload-placeholder').remove();
+                $('article > div.view-tracker').remove();
+                $('article > div.flex.flex-col').remove();
+
+                item.data.description = $('article').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `香港01 - ${data.issue.title}`,
+        description: data.meta.metaDesc,
+        link: data.issue.publishUrl,
+        item: list.map((item) => ({
+            title: item.data.title,
+            author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
+            description: `<img style="width: 100%" src="${item.data.mainImage.cdnUrl}" /><p>${item.data.description}</p>`,
+            pubDate: parseDate(item.data.lastModifyTime * 1000),
+            link: item.data.canonicalUrl,
+        })),
+    };
+};

package/lib/v2/hk01/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/channel/:id': ['hoilc', 'Fatpandac'],
+    '/hot': ['hoilc', 'Fatpandac'],
+    '/issue/:id': ['hoilc', 'Fatpandac'],
+    '/tag/:id': ['hoilc', 'Fatpandac'],
+    '/zone/:id': ['hoilc', 'Fatpandac'],
+};

package/lib/v2/hk01/radar.js

@@ -0,0 +1,37 @@
+module.exports = {
+    'hk01.com': {
+        _name: '香港01',
+        www: [
+            {
+                title: '最 Hit',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: ['/hot', '/'],
+                target: '/hk01/hot',
+            },
+            {
+                title: 'zone',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: '/zone/:id/:title',
+                target: '/hk01/zone/:id',
+            },
+            {
+                title: 'channel',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: '/channel/:id/:title',
+                target: '/hk01/channel/:id',
+            },
+            {
+                title: 'issue',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: '/issue/:id/:title',
+                target: '/hk01/issue/:id',
+            },
+            {
+                title: 'tag',
+                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
+                source: '/tag/:id/:title',
+                target: '/hk01/tag/:id',
+            },
+        ],
+    },
+};

package/lib/v2/hk01/router.js

@@ -0,0 +1,7 @@
+module.exports = function (router) {
+    router.get('/channel/:id', require('./channel'));
+    router.get('/hot', require('./hot'));
+    router.get('/issue/:id', require('./issue'));
+    router.get('/tag/:id', require('./tag'));
+    router.get('/zone/:id', require('./zone'));
+};

package/lib/{routes → v2}/hk01/tag.js

@@ -1,4 +1,5 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
@@ -15,7 +16,7 @@ module.exports = async (ctx) => {
             title: item.data.title,
             author: item.data.authors && item.data.authors.map((e) => e.publishName).join(', '),
             description: `<p>${item.data.description}</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`,
-            pubDate: new Date(item.data.lastModifyTime * 1000),
+            pubDate: parseDate(item.data.lastModifyTime * 1000),
             link: item.data.canonicalUrl,
         })),
     };

package/lib/{routes → v2}/hk01/zone.js

@@ -1,4 +1,5 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
@@ -19,12 +20,12 @@ module.exports = async (ctx) => {
                 case 1:
                     author = item.data.authors && item.data.authors.map((e) => e.publishName).join(', ');
                     description = `<p>${item.data.description}</p><img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`;
-                    pubDate = new Date(item.data.lastModifyTime * 1000);
+                    pubDate = parseDate(item.data.lastModifyTime * 1000);
                     break;
                 case 2:
                     author = item.data.zonePublishName;
                     description = `${item.data.teaser.map((e) => '<p>' + e + '</p>').join('')}<img style="width: 100%" src="${item.data.mainImage.cdnUrl}" />`;
-                    pubDate = new Date();
+                    pubDate = parseDate(item.data.lastModifyTime * 1000);
                     break;
                 default:
                     break;

package/lib/v2/hotukdeals/index.js

@@ -24,7 +24,7 @@ module.exports = async (ctx) => {
                 item = $(item);
                 return {
                     title: item.find('.cept-tt').text(),
-                    description: `${item.find('.cept-thread-image-link').html()}<br>${item.find('.cept-vote-temp').html()}<br>${item.find('.overflow--fade').html()}<br>${item.find('.cept-description-container').html()}`,
+                    description: `${item.find('.thread-listImgCell').html()}<br>${item.find('.cept-vote-temp').html()}<br>${item.find('.overflow--fade').html()}<br>${item.find('.threadGrid-body .userHtml').html()}`,
                     link: item.find('.cept-tt').attr('href'),
                 };
             })

package/lib/v2/ieee/journal.js

@@ -0,0 +1,78 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const punumber = ctx.params.journal;
+    const sortType = ctx.params.sortType ?? 'vol-only-seq';
+    const host = 'https://ieeexplore.ieee.org';
+    const jrnlUrl = `${host}/xpl/mostRecentIssue.jsp?punumber=${punumber}`;
+
+    const response = await got(`${host}/rest/publication/home/metadata?pubid=${punumber}`, {
+        cookieJar,
+    }).json();
+    const volume = response.currentIssue.volume;
+    const isnumber = response.currentIssue.issueNumber;
+    const jrnlName = response.displayTitle;
+
+    const response2 = await got
+        .post(`${host}/rest/search/pub/${punumber}/issue/${isnumber}/toc`, {
+            cookieJar,
+            json: {
+                punumber,
+                isnumber,
+                sortType,
+                rowsPerPage: '100',
+            },
+        })
+        .json();
+    let list = response2.records.map((item) => {
+        const $2 = cheerio.load(item.articleTitle);
+        const title = $2.text();
+        const link = item.htmlLink;
+        const doi = item.doi;
+        let authors = 'Do not have author';
+        if (item.hasOwnProperty('authors')) {
+            authors = item.authors.map((itemAuth) => itemAuth.preferredName).join('; ');
+        }
+        let abstract = '';
+        item.hasOwnProperty('abstract') ? (abstract = item.abstract) : (abstract = '');
+        return {
+            title,
+            link,
+            authors,
+            doi,
+            volume,
+            abstract,
+        };
+    });
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    list = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (item.abstract !== '') {
+                    const response3 = await got(`${host}${item.link}`);
+                    const { abstract } = JSON.parse(response3.body.match(/metadata=(.*);/)[1]);
+                    const $3 = cheerio.load(abstract);
+                    item.abstract = $3.text();
+                    item.description = renderDesc(item);
+                }
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: jrnlName,
+        link: jrnlUrl,
+        item: list,
+    };
+};

package/lib/v2/ieee/maintainer.js

@@ -1,4 +1,4 @@
 module.exports = {
-    '/:journal/latest/date/:sortType?': ['Derekmini', 'auto-bot-ty'],
-    '/:journal/latest/vol/:sortType?': ['Derekmini', 'auto-bot-ty'],
+    '/journal/:journal/:sortType?': ['Derekmini', 'auto-bot-ty'],
+    '/journal/:journal/recent/:sortType?': ['Derekmini', 'auto-bot-ty'],
 };

package/lib/v2/ieee/radar.js

@@ -3,16 +3,16 @@ module.exports = {
         _name: 'IEEE',
         www: [
             {
-                title: 'latestVolume',
+                title: 'Journal',
                 docs: 'https://docs.rsshub.app/journal.html#ieee-xplore',
                 source: '/*',
-                target: (params, url) => `/ieee/${new URL(url).searchParams.get('punumber')}/latest/vol`,
+                target: (params, url) => `/ieee/journal/${new URL(url).searchParams.get('punumber')}`,
             },
             {
-                title: 'latestDate',
+                title: 'Recent',
                 docs: 'https://docs.rsshub.app/journal.html#ieee-xplore',
                 source: '/*',
-                target: (params, url) => `/ieee/${new URL(url).searchParams.get('punumber')}/latest/date`,
+                target: (params, url) => `/ieee/journal/${new URL(url).searchParams.get('punumber')}/recent`,
             },
         ],
     },

package/lib/v2/ieee/{latestdate.js → recent.js}

@@ -1,21 +1,17 @@
 const got = require('@/utils/got');
-const config = require('@/config').value;
+const cheerio = require('cheerio');
 const path = require('path');
 const { art } = require('@/utils/render');
 
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
 module.exports = async (ctx) => {
-    const host = 'https://ieeexplore.ieee.org';
     const punumber = ctx.params.journal;
     const sortType = ctx.params.sortType ?? 'vol-only-seq';
-    const hostUrl = host.concat('/xpl/mostRecentIssue.jsp?punumber=', punumber);
-    const headers = {
-        Accept: 'application/json, text/plain, */*',
-        'Accept-Encoding': 'gzip, deflate, br',
-        'Accept-Language': 'zh-CN,zh;q=0.9',
-        Host: host.replace('https://', ''),
-        'User-Agent': config.ua,
-        Referer: hostUrl,
-    };
+    const host = 'https://ieeexplore.ieee.org';
+    const jrnlUrl = `${host}/xpl/mostRecentIssue.jsp?punumber=${punumber}`;
+
     const date = new Date();
     const year = date.getFullYear();
     const month = date.getMonth() + 1;
@@ -32,29 +28,28 @@ module.exports = async (ctx) => {
         endYM = year + '' + month;
     }
 
-    const resp = await got({
-        method: 'get',
-        url: host.concat('/rest/publication/home/metadata?pubid=', punumber),
-        headers,
+    const response = await got(`${host}/rest/publication/home/metadata?pubid=${punumber}`, {
+        cookieJar,
     }).json();
-    const volume = resp.currentIssue.volume;
-    const isnumber = resp.currentIssue.issueNumber;
-    const jrnlName = resp.displayTitle;
+    const volume = response.currentIssue.volume;
+    const isnumber = response.currentIssue.issueNumber;
+    const jrnlName = response.displayTitle;
 
-    const response = await got({
-        method: 'post',
-        url: host.concat('/rest/search/pub/', punumber, '/issue/', isnumber, '/toc'),
-        headers,
-        json: {
-            punumber,
-            isnumber,
-            sortType,
-            rowsPerPage: '100',
-            ranges: [strYM + `01_` + endYM + `31_Search Latest Date`],
-        },
-    }).json();
-    const list = response.records.map((item) => {
-        const title = item.articleTitle;
+    const response2 = await got
+        .post(`${host}/rest/search/pub/${punumber}/issue/${isnumber}/toc`, {
+            cookieJar,
+            json: {
+                punumber,
+                isnumber,
+                sortType,
+                rowsPerPage: '100',
+                ranges: [strYM + `01_` + endYM + `31_Search Latest Date`],
+            },
+        })
+        .json();
+    let list = response2.records.map((item) => {
+        const $2 = cheerio.load(item.articleTitle);
+        const title = $2.text();
         const link = item.htmlLink;
         const doi = item.doi;
         let authors = 'Do not have author';
@@ -77,17 +72,14 @@ module.exports = async (ctx) => {
         art(path.join(__dirname, 'templates/description.art'), {
             item,
         });
-    await Promise.all(
+    list = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 if (item.abstract !== '') {
-                    const itemResponse = await got({
-                        method: 'get',
-                        url: host.concat(item.link),
-                    });
-                    const content = itemResponse.body.match(/metadata=(.*);/)[1];
-                    const { abstract } = JSON.parse(content);
-                    item.abstract = abstract;
+                    const response3 = await got(`${host}${item.link}`);
+                    const { abstract } = JSON.parse(response3.body.match(/metadata=(.*);/)[1]);
+                    const $3 = cheerio.load(abstract);
+                    item.abstract = $3.text();
                     item.description = renderDesc(item);
                 }
                 return item;
@@ -96,8 +88,8 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: jrnlName.concat(' - Latest'),
-        link: hostUrl,
+        title: `${jrnlName} - Recent`,
+        link: jrnlUrl,
         item: list,
     };
 };

package/lib/v2/ieee/router.js

@@ -1,4 +1,6 @@
 module.exports = function (router) {
-    router.get('/:journal/latest/date/:sortType?', require('./latestdate'));
-    router.get('/:journal/latest/vol/:sortType?', require('./latestvol'));
+    router.get('/:journal/latest/vol/:sortType?', require('./journal')); // deprecated
+    router.get('/:journal/latest/date/:sortType?', require('./recent')); // deprecated
+    router.get('/journal/:journal/:sortType?', require('./journal'));
+    router.get('/journal/:journal/recent/:sortType?', require('./recent'));
 };

package/lib/v2/ieee/templates/description.art

@@ -1,11 +1,11 @@
 <p>
-    <span><big>{{ item.title }}</big></span><br />
+    <span><big>{{ item.title }}</big></span><br>
 </p>
 <p>
-    <span><small><i>{{ item.authors }}</i></small></span><br />
-    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br />
-    <span><small><i>Volume: {{ item.volume }}</i></small></span><br />
+    <span><small><i>{{ item.authors }}</i></small></span><br>
+    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br>
+    <span><small><i>Volume {{ item.volume }}</i></small></span><br>
 </p>
 <p>
-    <span>{{ item.abstract }}</span><br />    
+    <span>{{ item.abstract }}</span><br>
 </p>

package/lib/v2/ielts/index.js

@@ -11,8 +11,13 @@ module.exports = async (ctx) => {
         async () => {
             const browser = await require('@/utils/puppeteer')({ stealth: true });
             const page = await browser.newPage();
-
-            await page.goto(targetUrl);
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
+            await page.goto(targetUrl, {
+                waitUntil: 'domcontentloaded',
+            });
             await page.waitForSelector('div.container');
 
             const html = await page.evaluate(() => document.documentElement.innerHTML);

package/lib/v2/keep/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/user/:id': ['Dectinc', 'DIYgod'],
+};

package/lib/v2/keep/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'gotokeep.com': {
+        _name: 'Keep',
+        '.': [
+            {
+                title: '用户运动日记',
+                docs: 'https://docs.rsshub.app/social-media.html#keep',
+                source: '/users/:id',
+                target: '/keep/user/:id',
+            },
+        ],
+    },
+};

package/lib/v2/keep/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/user/:id', require('./user'));
+};

package/lib/v2/keep/templates/user.art

@@ -0,0 +1,16 @@
+ 项目：
+{{ if item.meta.name === item.meta.workoutName }}
+{{ item.meta.name }}
+{{ else }}
+{{ item.meta.name }} - {{ item.meta.workoutName }}
+{{ /if }}
+<br>
+时长：{{ minute }}分{{ second }}秒
+{{ if item.content }}
+<br>
+备注：{{ item.content }}
+{{ /if }}
+{{ if imagesTpl }}
+<br>
+{{@ imagesTpl }}
+{{ /if }}

package/lib/{routes → v2}/keep/user.js

@@ -1,4 +1,6 @@
-const got = require('../../utils/got');
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
@@ -11,6 +13,11 @@ module.exports = async (ctx) => {
         },
     });
 
+    // check user have post or not
+    if (response.data.data.entries.length === 0) {
+        throw Error('该用户运动日记为空');
+    }
+
     response.data.data.entries[0] = response.data.data.entries[0].entries;
     const data = response.data.data.entries.reduce((all, current) => all.concat(current.entries));
 
@@ -38,9 +45,12 @@ module.exports = async (ctx) => {
                     title: item.meta.title.trim(),
                     pubDate: item.created,
                     link: `https://show.gotokeep.com/entries/${item.id}`,
-                    description: `项目：${item.meta.name === item.meta.workoutName ? item.meta.name : `${item.meta.name} - ${item.meta.workoutName}`}<br>时长：${minute}分${second}秒${item.content ? `<br>备注：${item.content}` : ''}${
-                        imagesTpl ? `<br>${imagesTpl}` : ''
-                    }`,
+                    description: art(path.join(__dirname, 'templates/user.art'), {
+                        item,
+                        minute,
+                        second,
+                        imagesTpl,
+                    }),
                 };
             }),
     };

package/lib/v2/mdpi/journal.js

@@ -0,0 +1,65 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const journal = ctx.params.journal;
+    const host = 'https://www.mdpi.com';
+    const jrnlUrl = `${host}/journal/${journal}`;
+
+    const response = await got(jrnlUrl, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+    const jrnlName = $('.journal__description').find('h1').text();
+    const issueUrl = `${host}${$('.side-menu-ul').find('a').eq(1).attr('href')}`;
+
+    const response2 = await got(issueUrl, {
+        cookieJar,
+    });
+    const $2 = cheerio.load(response2.data);
+    const issue = $2('.content__container').find('h1').text().trim();
+    const list = $2('.article-item')
+        .map((_, item) => {
+            const title = $2(item).find('.title-link').text();
+            const link = `${host}${$2(item).find('.title-link').attr('href')}`;
+            const authors = $2(item).find('.authors').find('.inlineblock').text();
+            const doiLink = $2(item).find('.color-grey-dark').find('a').attr('href');
+            let doi = '';
+            if (typeof doiLink !== 'undefined') {
+                doi = doiLink.replace('https://doi.org/', '');
+            }
+            $2(item).find('.abstract-full').find('a').remove();
+            const abstract = $2(item).find('.abstract-full').text().trim();
+            const img = `${host}${$2(item).find('.openpopupgallery').find('img').attr('data-src')}`;
+            return {
+                title,
+                authors,
+                link,
+                doi,
+                abstract,
+                issue,
+                img,
+            };
+        })
+        .get();
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    const items = list.map((item) => {
+        item.description = renderDesc(item);
+        return item;
+    });
+
+    ctx.state.data = {
+        title: jrnlName,
+        link: jrnlUrl,
+        item: items,
+    };
+};

package/lib/v2/mdpi/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:journal': ['Derekmini'],
+};

package/lib/v2/mdpi/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'mdpi.com': {
+        _name: 'MDPI',
+        www: [
+            {
+                title: 'Journal',
+                docs: 'https://docs.rsshub.app/journal.html#MDPI',
+                source: '/journal/:journal',
+                target: '/mdpi/:journal',
+            },
+        ],
+    },
+};

package/lib/v2/mdpi/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:journal', require('./journal'));
+};