rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/lib/v2/elsevier/journal.js

@@ -0,0 +1,72 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const journal = ctx.params.journal;
+    const host = 'https://www.sciencedirect.com';
+    const jrnlUrl = `${host}/journal/${journal}`;
+
+    const response = await got(jrnlUrl, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+    const jrnlName = $('.anchor.js-title-link').text();
+    const issueUrl = `${host}${$('.anchor.js-volume').attr('href')}`;
+    let issue = '';
+    if (issueUrl.match('suppl') !== null) {
+        issue = 'Volume ' + issueUrl.match('vol/(.*)/suppl')[1];
+    } else if (issueUrl.match('issue') !== null) {
+        issue = 'Volume ' + issueUrl.match('vol/(.*)/issue')[1] + ' Issue ' + issueUrl.match('/issue/(.*)')[1];
+    }
+
+    const response2 = await got(issueUrl, {
+        cookieJar,
+    });
+    const $2 = cheerio.load(response2.data);
+    const list = $2('.js-article')
+        .map((_, item) => {
+            const title = $2(item).find('.js-article-title').text();
+            const authors = $2(item).find('.js-article__item__authors').text();
+            const link = $2(item).find('.article-content-title').attr('href');
+            const id = $2(item).find('.article-content-title').attr('id');
+            return {
+                title,
+                link,
+                id,
+                authors,
+                issue,
+            };
+        })
+        .get();
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response3 = await got(`${host}/science/article/pii/${item.id}`, {
+                    cookieJar,
+                });
+                const $3 = cheerio.load(response3.data);
+                $3('.section-title').remove();
+                item.doi = $3('.doi').attr('href').replace('https://doi.org/', '');
+                item.abstract = $3('.abstract.author').text();
+                item.description = renderDesc(item);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: jrnlName,
+        link: jrnlUrl,
+        item: items,
+    };
+};

package/lib/v2/elsevier/maintainer.js

@@ -1,4 +1,4 @@
 module.exports = {
-    '/:journal/latest': ['Derekmini', 'sunwolf-swb'],
-    '/:journal/vol/:id': ['Derekmini', 'sunwolf-swb'],
+    '/:journal': ['Derekmini', 'sunwolf-swb'],
+    '/:journal/:issue': ['Derekmini', 'sunwolf-swb'],
 };

package/lib/v2/elsevier/radar.js

@@ -3,16 +3,16 @@ module.exports = {
         _name: 'Elsevier',
         www: [
             {
-                title: 'latest',
+                title: 'Journal',
                 docs: 'https://docs.rsshub.app/journal.html#Elsevier',
                 source: '/journal/:journal/*',
-                target: '/elsevier/:journal/latest',
+                target: '/elsevier/:journal',
             },
             {
-                title: 'volume',
+                title: 'Issue',
                 docs: 'https://docs.rsshub.app/journal.html#Elsevier',
-                source: '/journal/:journal/vol/:id',
-                target: '/elsevier/:journal/vol/:id',
+                source: '/journal/:journal/vol/:issue',
+                target: '/elsevier/:journal/:issue',
             },
         ],
     },

package/lib/v2/elsevier/router.js

@@ -1,4 +1,6 @@
 module.exports = function (router) {
-    router.get('/:journal/latest', require('./latest'));
-    router.get('/:journal/vol/:id', require('./volume'));
+    router.get('/:journal/latest', require('./journal')); // deprecated
+    router.get('/:journal/vol/:issue', require('./issue')); // deprecated
+    router.get('/:journal', require('./journal'));
+    router.get('/:journal/:issue', require('./issue'));
 };

package/lib/v2/elsevier/templates/description.art

@@ -1,11 +1,11 @@
 <p>
-    <span><big>{{ item.title }}</big></span><br />
+    <span><big>{{ item.title }}</big></span><br>
 </p>
 <p>
-    <span><small><i>{{ item.authors }}</i></small></span><br />
-    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br />
-    <span><small><i>{{ item.volume }}</i></small></span><br />
+    <span><small><i>{{ item.authors }}</i></small></span><br>
+    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br>
+    <span><small><i>{{ item.issue }}</i></small></span><br>
 </p>
 <p>
-    <span>{{ item.abstract }}</span><br />    
+    <span>{{ item.abstract }}</span><br>
 </p>

package/lib/v2/epicgames/index.js

@@ -11,6 +11,7 @@ module.exports = async (ctx) => {
     const rootUrl = 'https://store.epicgames.com';
     const currentUrl = `${rootUrl}/${locale}/free-games?lang=${locale}`;
     const apiUrl = `https://store-site-backend-static-ipv4.ak.epicgames.com/freeGamesPromotions?locale=${locale}&country=${country}&allowCountries=${country}`;
+    const contentBaseUrl = `https://store-content-ipv4.ak.epicgames.com/api/${locale}/content`;
 
     const response = await got({
         method: 'get',
@@ -27,20 +28,53 @@ module.exports = async (ctx) => {
                 dayjs(item.promotions.promotionalOffers[0].promotionalOffers[0].startDate) <= now &&
                 dayjs(item.promotions.promotionalOffers[0].promotionalOffers[0].endDate) > now
         )
-        .map((item) => ({
-            title: item.title,
-            author: item.seller.name,
-            link: `${rootUrl}/${locale}/p/${item.catalogNs.mappings.length > 0 ? item.catalogNs.mappings[0].pageSlug : item.offerMappings.length > 0 ? item.offerMappings[0].pageSlug : item.urlSlug}`,
-            description: art(path.join(__dirname, 'templates/description.art'), {
-                description: item.description,
-                image: item.keyImages[0].url,
-            }),
-            pubDate: parseDate(item.promotions.promotionalOffers[0].promotionalOffers[0].startDate),
-        }));
+        .map(async (item) => {
+            let link = `${rootUrl}/${locale}/p/`;
+            let contentUrl = `${contentBaseUrl}/products/`;
+            let isBundles = false;
+            item.categories.some((category) => {
+                if (category.path === 'bundles') {
+                    link = `${rootUrl}/${locale}/bundles/`;
+                    isBundles = true;
+                    contentUrl = `${contentBaseUrl}/bundles/`;
+                    return true;
+                }
+                return false;
+            });
+            const linkSlug = item.catalogNs.mappings.length > 0 ? item.catalogNs.mappings[0].pageSlug : item.offerMappings.length > 0 ? item.offerMappings[0].pageSlug : item.productSlug ? item.productSlug : item.urlSlug;
+            link += linkSlug;
+            contentUrl += linkSlug;
+            let description = item.description;
+            if (item.offerType !== 'BASE_GAME') {
+                const contentResp = await got({
+                    method: 'get',
+                    url: contentUrl,
+                });
+                description = isBundles ? contentResp.data.data.about.shortDescription : contentResp.data.pages[0].data.about.shortDescription;
+            }
 
+            let image = item.keyImages[0].url;
+            item.keyImages.some((keyImage) => {
+                if (keyImage.type === 'DieselStoreFrontWide') {
+                    image = keyImage.url;
+                    return true;
+                }
+                return false;
+            });
+            return {
+                title: item.title,
+                author: item.seller.name,
+                link,
+                description: art(path.join(__dirname, 'templates/description.art'), {
+                    description,
+                    image,
+                }),
+                pubDate: parseDate(item.promotions.promotionalOffers[0].promotionalOffers[0].startDate),
+            };
+        });
     ctx.state.data = {
         title: 'Epic Games Store - Free Games',
         link: currentUrl,
-        item: items,
+        item: await Promise.all(items),
     };
 };

package/lib/v2/feng/forum.js

@@ -0,0 +1,42 @@
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, getForumMeta, getThreads, getThread } = require('./utils');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const topicId = Number(ctx.params.id);
+    const { type = 'all' } = ctx.params;
+
+    const forumMeta = await getForumMeta(topicId, ctx);
+    const topicMeta = forumMeta.dataList.find((data) => data.topicId === topicId);
+    const threads = (await getThreads(topicId, type, ctx)).data.dataList.map((data) => ({
+        title: data.title,
+        pubDate: parseDate(data.dateline * 1000),
+        author: data.userBaseInfo.userName,
+        link: `${baseUrl}/post/${data.tid}`,
+        tid: data.tid,
+    }));
+
+    const posts = await Promise.all(
+        threads.map(async (item) => {
+            const thread = await getThread(item.tid, topicId, ctx);
+            if (thread.status.code === 0) {
+                const img = art(path.join(__dirname, 'templates/img.art'), {
+                    images: thread.data.thread.fengTalkImage.length ? thread.data.thread.fengTalkImage : undefined,
+                });
+                item.description = thread.data.thread.message + img;
+            } else {
+                item.description = art(path.join(__dirname, 'templates/deleted.art'), {});
+            }
+            delete item.tid;
+            return item;
+        })
+    );
+
+    ctx.state.data = {
+        title: `${topicMeta.topicName} - 社区 - 威锋 - 千万果粉大本营`,
+        description: topicMeta.topicDescription,
+        link: `${baseUrl}/forum/${topicId}`,
+        item: posts,
+    };
+};

package/lib/v2/feng/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'feng.com': {
+        _name: '威锋',
+        '.': [
+            {
+                title: '社区',
+                docs: 'https://docs.rsshub.app/bbs.html#wei-feng',
+                source: ['/forum/photo/:id', '/forum/:id'],
+                target: '/feng/forum/:id',
+            },
+        ],
+    },
+};

package/lib/v2/feng/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/forum/:id/:type?', require('./forum'));
+};

package/lib/v2/feng/templates/deleted.art

@@ -0,0 +1,2 @@
+<img src="https://www.feng.com/_nuxt/img/yishanchu.368ead2.png" alt="威锋">
+<div align="center">帖子已被删除</div>

package/lib/v2/feng/templates/img.art

@@ -0,0 +1,6 @@
+{{ if images }}
+<br>
+{{ each images }}
+<img src="{{ $value.split('?')[0] }}" alt="">
+{{ /each }}
+{{ /if }}

package/lib/v2/feng/utils.js

@@ -0,0 +1,80 @@
+const CryptoJS = require('crypto-js');
+const got = require('@/utils/got');
+const config = require('@/config').value;
+const apiUrl = 'https://api.wfdata.club';
+const baseUrl = 'https://www.feng.com';
+const KEY = '2b7e151628aed2a6';
+
+// https://juejin.cn/post/6844904066468806664
+const getXRequestId = (apiUrl) => {
+    const path = new URL(apiUrl).pathname; // split search params
+    const plainText = CryptoJS.enc.Utf8.parse('url=' + path + '$time=' + Date.now() + '000000');
+    const iv = CryptoJS.enc.Utf8.parse(KEY);
+    const encrypted = CryptoJS.AES.encrypt(plainText, iv, {
+        iv,
+        mode: CryptoJS.mode.CBC,
+        padding: CryptoJS.pad.Pkcs7,
+    }).toString();
+    return encrypted;
+};
+
+const getCategory = (topicId, ctx) => {
+    const url = `${apiUrl}/v1/topic/category`;
+    return ctx.cache.tryGet(
+        url,
+        async () => {
+            const response = await got(url, {
+                headers: {
+                    Referer: `${baseUrl}/forum/${topicId}`,
+                    'X-Request-Id': getXRequestId(url),
+                },
+            });
+            return response.data;
+        },
+        config.cache.routeExpire,
+        false
+    );
+};
+
+const getForumMeta = async (topicId, ctx) => {
+    const categoryData = await getCategory(topicId, ctx);
+    return Object.values(categoryData.data.dataList).find((item) => item.dataList.find((i) => i.topicId === topicId));
+};
+
+const getThreads = (topicId, type, ctx) => {
+    const url = `${apiUrl}/v1/topic/${topicId}/thread?topicId=${topicId}&type=${type}&pageCount=50&order=${type === 'newest' ? 'replyTime' : 'postTime'}&page=1`;
+    return ctx.cache.tryGet(
+        url,
+        async () => {
+            const response = await got(url, {
+                headers: {
+                    Referer: `${baseUrl}/forum/${topicId}`,
+                    'X-Request-Id': getXRequestId(url),
+                },
+            });
+            return response.data;
+        },
+        config.cache.routeExpire,
+        false
+    );
+};
+
+const getThread = (tid, topicId, ctx) => {
+    const url = `${apiUrl}/v1/thread/${tid}`;
+    return ctx.cache.tryGet(url, async () => {
+        const response = await got(url, {
+            headers: {
+                Referer: `${baseUrl}/forum/${topicId}`,
+                'X-Request-Id': getXRequestId(url),
+            },
+        });
+        return response.data;
+    });
+};
+
+module.exports = {
+    baseUrl,
+    getForumMeta,
+    getThreads,
+    getThread,
+};

package/lib/v2/fx-markets/channel.js

@@ -0,0 +1,56 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const channel = ctx.params.channel;
+    const link = `https://www.fx-markets.com/${channel}`;
+    const html = (await got(link)).data;
+    const $ = cheerio.load(html);
+    const pageTitle = $('header.select-header > h1').text();
+    const title = `FX-Markets ${pageTitle}`;
+
+    const items = $('div#listings').children();
+    const articles = items
+        .map((i, el) => {
+            const $el = $(el);
+            const $titleEl = $el.find('h5 > a');
+            const articleURL = `https://www.fx-markets.com${$titleEl.attr('href')}`;
+            const articleTitle = $titleEl.attr('title');
+            return {
+                title: articleTitle,
+                link: articleURL,
+                pubDate: parseDate($el.find('time').text()),
+            };
+        })
+        .get();
+
+    const result = await Promise.all(
+        articles.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const res = await got(item.link);
+                const doc = cheerio.load(res.data);
+                // This script holds publish datetime info {"datePublished": "2022-05-12T08:45:04+01:00"}
+                const dateScript = doc('script[type="application/ld+json"]').get()[0].children[0].data;
+                const re = /"datePublished": "(?<dateTimePub>.*)"/;
+                const dateStr = re.exec(dateScript).groups.dateTimePub;
+                const pubDateTime = parseDate(dateStr, 'YYYY-MM-DDTHH:mm:ssZ');
+                // Exclude hidden print message
+                item.description = doc('div.article-page-body-content:not(.print-access-info)').html();
+                return {
+                    title: item.title,
+                    link: item.link,
+                    description: item.description,
+                    // if we fail to get accurate publish date time, show date only from article link on index page.
+                    pubDate: pubDateTime ? pubDateTime : item.pubDate,
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title,
+        link,
+        item: result,
+    };
+};

package/lib/v2/fx-markets/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/fx-markets/:channel': ['mikkkee'],
+};

package/lib/v2/fx-markets/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'fx-markets.com': {
+        _name: 'FX-Markets',
+        '.': [
+            {
+                title: 'Trading',
+                docs: 'https://docs.rsshub.app/finance.html#fx-markets',
+                source: '/trading',
+                target: '/fx-markets/trading',
+            },
+            {
+                title: 'Infrastructure',
+                docs: 'https://docs.rsshub.app/finance.html#fx-markets',
+                source: '/infrastructure',
+                target: '/fx-markets/infrastructure',
+            },
+            {
+                title: 'Tech and Data',
+                docs: 'https://docs.rsshub.app/finance.html#fx-markets',
+                source: '/tech-and-data',
+                target: '/fx-markets/tech-and-data',
+            },
+            {
+                title: 'Regulation',
+                docs: 'https://docs.rsshub.app/finance.html#fx-markets',
+                source: '/regulation',
+                target: '/fx-markets/regulation',
+            },
+        ],
+    },
+};

package/lib/v2/fx-markets/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:channel', require('./channel'));
+};

package/lib/v2/gcores/category.js

@@ -15,7 +15,7 @@ module.exports = async (ctx) => {
     const list = $('.original.am_card.original-normal')
         .map(function () {
             const item = {
-                url: $(this).find('a.am_card_content.original_content').attr('href'),
+                url: $(this).find('.am_card_inner>a').attr('href'),
                 title: $(this).find('h3.am_card_title').text(),
                 category: $(this).find('span.original_category>a').text(),
             };
@@ -23,6 +23,10 @@ module.exports = async (ctx) => {
         })
         .get();
 
+    if (list.length > 0 && list.every((item) => item.url === undefined)) {
+        throw new Error('Article URL not found! Please submit an issue on GitHub.');
+    }
+
     const out = await Promise.all(
         list.map((item) => {
             const articleUrl = `https://www.gcores.com${item.url}`;

package/lib/v2/gcores/tag.js

@@ -16,7 +16,7 @@ module.exports = async (ctx) => {
     const list = $('.original.am_card.original-normal')
         .map(function () {
             const item = {
-                url: $(this).find('a.am_card_content.original_content').attr('href'),
+                url: $(this).find('.am_card_inner>a').attr('href'),
                 title: $(this).find('h3.am_card_title').text(),
                 category: $(this).find('span.original_category>a').text(),
             };
@@ -24,6 +24,10 @@ module.exports = async (ctx) => {
         })
         .get();
 
+    if (list.length > 0 && list.every((item) => item.url === undefined)) {
+        throw new Error('Article URL not found! Please submit an issue on GitHub.');
+    }
+
     const out = await Promise.all(
         list.map((item) => {
             const articleUrl = `https://www.gcores.com${item.url}`;

package/lib/v2/getitfree/index.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? '';
+
+    const rootUrl = 'https://www.getitfree.cn';
+    const currentUrl = `${rootUrl}${category ? `/category/${category}` : ''}/page/1`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('a.post-title')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('h3, footer').remove();
+
+                item.description = content('.post-content-text').html();
+                item.pubDate = parseDate(content('time.entry-date').first().attr('datetime'));
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/getitfree/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category?': ['nczitzk'],
+};

package/lib/v2/getitfree/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'getitfree.cn': {
+        _name: '正版中国',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/shopping.html#zheng-ban-zhong-guo-fen-lei',
+                source: ['/category/:category', '/'],
+                target: '/getitfree/:category?',
+            },
+        ],
+    },
+};

package/lib/v2/getitfree/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:category?', require('./index'));
+};

package/lib/v2/gitee/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/commits/:owner/:repo': ['TonyRL'],
+    '/events/:owner/:repo': ['TonyRL'],
+    '/events/:username': ['TonyRL'],
+    '/releases/:owner/:repo': ['TonyRL'],
+};

package/lib/v2/gitee/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'gitee.com': {
+        _name: 'Gitee',
+        '.': [
+            {
+                title: '仓库 Releases',
+                link: 'https://docs.rsshub.app',
+                source: ['/:owner/:repo/releases'],
+                target: '/gitee/releases/:owner/:repo',
+            },
+            {
+                title: '仓库提交',
+                link: 'https://docs.rsshub.app',
+                source: ['/:owner/:repo/commits'],
+                target: '/gitee/commits/:owner/:repo',
+            },
+            {
+                title: '用户公开动态',
+                link: 'https://docs.rsshub.app',
+                source: ['/:username'],
+                target: '/gitee/events/:username',
+            },
+            {
+                title: '仓库动态',
+                link: 'https://docs.rsshub.app',
+                source: ['/:owner/:repo'],
+                target: '/gitee/events/:owner/:repo',
+            },
+        ],
+    },
+};

package/lib/v2/gitee/repos/commits.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const { owner, repo } = ctx.params;
+
+    const apiUrl = `https://gitee.com/api/v5/repos/${owner}/${repo}/commits`;
+    const response = await ctx.cache.tryGet(
+        apiUrl,
+        async () =>
+            (
+                await got(apiUrl, {
+                    searchParams: {
+                        access_token: config.gitee.access_token ? config.gitee.access_token : undefined,
+                        per_page: ctx.query.limit ? Number(ctx.query.limit) : 100,
+                        direction: 'desc',
+                    },
+                })
+            ).data
+    );
+
+    const items = response.map((item) => ({
+        title: md.renderInline(item.commit.message),
+        description: md.render(item.commit.message),
+        author: item.author?.login || item.commit.author.name,
+        pubDate: parseDate(item.commit.author.date),
+        guid: item.sha,
+        link: item.html_url,
+    }));
+
+    ctx.state.data = {
+        title: `${owner}/${repo} - 提交`,
+        link: `https://gitee.com/${owner}/${repo}/commits`,
+        item: items,
+    };
+};