rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/v2/ehentai/ehapi.js

@@ -0,0 +1,172 @@
+const got = require('@/utils/got');
+const logger = require('@/utils/logger');
+const cheerio = require('cheerio');
+const path = require('path');
+const config = require('@/config').value.ehentai;
+
+const headers = {};
+const has_cookie = config.ipb_member_id && config.ipb_pass_hash && config.sk;
+const from_ex = has_cookie && config.igneous;
+if (has_cookie) {
+    if (from_ex) {
+        const { ipb_member_id, ipb_pass_hash, sk, igneous } = config;
+        headers.cookie = `ipb_member_id=${ipb_member_id};ipb_pass_hash=${ipb_pass_hash};sk=${sk};igneous=${igneous}`;
+    } else {
+        const { ipb_member_id, ipb_pass_hash, sk } = config;
+        headers.cookie = `ipb_member_id=${ipb_member_id};ipb_pass_hash=${ipb_pass_hash};sk=${sk}`;
+    }
+}
+
+function ehgot(url) {
+    if (from_ex) {
+        return got({ method: 'get', url: `https://exhentai.org/${url}`, headers });
+    } else {
+        return got({ method: 'get', url: `https://e-hentai.org/${url}`, headers });
+    }
+}
+
+function ehgot_thumb(cache, thumb_url) {
+    return cache.tryGet(thumb_url, async () => {
+        try {
+            const buffer = await got({ method: 'get', url: thumb_url, headers });
+            const data = new Buffer.from(buffer.rawBody).toString('base64');
+            const ext = path.extname(thumb_url).slice(1);
+            return `data:image/${ext};base64,${data}`;
+        } catch (e) {
+            logger.warn('Cannot download thumbnail: ' + thumb_url, e);
+            return thumb_url;
+        }
+    });
+}
+
+async function parsePage(cache, data, get_bittorrent = false, embed_thumb = false) {
+    const $ = cheerio.load(data);
+    const table = $('table[class^="itg glt"] tbody');
+    if (!table) {
+        return [];
+    }
+
+    async function parseTableRow(cache, element) {
+        const el = $(element);
+        const title = el.find('div.glink').html();
+        const pubDate = el.find('div[id^="posted_"]').html();
+        // match "Minimal", "Minimal+" and "Compact"
+        let el_a = el.find('td[class^="gl3"] a');
+        let el_img = el.find('td[class^=gl2] div.glthumb div img');
+        // if not, match "Extended"
+        if (!(el_a.length && el_img.length)) {
+            el_a = el.find('td[class^="gl1"] a');
+            el_img = el_a.find('img');
+        }
+        const link = el_a.attr('href');
+        let thumbnail = el_img.attr('data-src') ? el_img.attr('data-src') : el_img.attr('src');
+        if (config.img_proxy && thumbnail) {
+            const url = new URL(thumbnail);
+            thumbnail = config.img_proxy + url.pathname + url.search;
+        }
+        if (embed_thumb && thumbnail) {
+            thumbnail = await ehgot_thumb(cache, thumbnail);
+        }
+        const description = `<img src='${thumbnail}' alt='thumbnail'>`;
+        if (title && link) {
+            const item = { title, description, pubDate, link };
+            if (get_bittorrent) {
+                const el_down = el.find('div.gldown');
+                const bittorrent_page_url = el_down.find('a').attr('href');
+                if (bittorrent_page_url) {
+                    const bittorrent_url = await getBittorrent(cache, bittorrent_page_url);
+                    if (bittorrent_url) {
+                        item.enclosure_url = bittorrent_url;
+                        item.enclosure_type = 'application/x-bittorrent';
+                        item.bittorrent_page_url = bittorrent_page_url;
+                    }
+                }
+            }
+            return item;
+        }
+    }
+
+    const item_Promises = [];
+    table.children('tr').each((index, element) => {
+        item_Promises.push(parseTableRow(cache, element));
+    });
+    const items_with_null = await Promise.all(item_Promises);
+
+    const items = [];
+    for (const item of items_with_null) {
+        if (item) {
+            items.push(item);
+        }
+    }
+    return items;
+}
+
+let p = '';
+
+function getBittorrent(cache, bittorrent_page_url) {
+    return cache.tryGet(bittorrent_page_url, async () => {
+        try {
+            const response = await got({ method: 'get', url: bittorrent_page_url, headers });
+            const $ = cheerio.load(response.data);
+            const el_forms = $('form').get();
+            let bittorrent_url = undefined;
+            for (const el_form of el_forms) {
+                const el_a = $(el_form).find('a');
+                const onclick = el_a.attr('onclick');
+                if (onclick) {
+                    const match = onclick.match(/'(.*?)'/);
+                    if (match) {
+                        bittorrent_url = match[1];
+                        const match_p = bittorrent_url.match(/torrent\?p=(.*?)$/);
+                        if (match_p) {
+                            p = match_p[1];
+                        }
+                    }
+                }
+            }
+            return bittorrent_url;
+        } catch {
+            return undefined;
+        }
+    });
+}
+
+function updateBittorrent_url(cache, items) {
+    // 下种子文件需要动态密码，密码每几次请求就更新一次
+    for (const item of items) {
+        if (item.enclosure_url) {
+            item.enclosure_url = item.enclosure_url.replace(/torrent\?p=.*$/, `torrent?p=${p}`);
+            cache.set(item.bittorrent_page_url, item.enclosure_url);
+        }
+    }
+    return items;
+}
+
+async function gatherItemsByPage(cache, url, get_bittorrent = false, embed_thumb = false) {
+    const response = await ehgot(url);
+    const items = await parsePage(cache, response.data, get_bittorrent, embed_thumb);
+    return updateBittorrent_url(cache, items);
+}
+
+async function getFavoritesItems(cache, page, favcat, inline_set, get_bittorrent = false, embed_thumb = false) {
+    const response = await ehgot(`favorites.php?favcat=${favcat}&inline_set=${inline_set}`);
+    if (parseInt(page) === 0) {
+        const items = await parsePage(cache, response.data, get_bittorrent, embed_thumb);
+        return updateBittorrent_url(cache, items);
+    }
+    return gatherItemsByPage(cache, `favorites.php?page=${page}&favcat=${favcat}`, get_bittorrent, embed_thumb);
+}
+
+function getSearchItems(cache, params, page = undefined, get_bittorrent = false, embed_thumb = false) {
+    if (page) {
+        return gatherItemsByPage(cache, `?page=${page}&${params}`, get_bittorrent, embed_thumb);
+    } else {
+        return gatherItemsByPage(cache, `?${params}`, get_bittorrent, embed_thumb);
+    }
+}
+
+function getTagItems(cache, tag, page, get_bittorrent = false, embed_thumb = false) {
+    return gatherItemsByPage(cache, `tag/${tag}/${page}`, get_bittorrent, embed_thumb);
+}
+
+module.exports = { getFavoritesItems, getSearchItems, getTagItems, has_cookie, from_ex };

package/lib/v2/ehentai/favorites.js

@@ -0,0 +1,28 @@
+const EhAPI = require('./ehapi');
+
+module.exports = async (ctx) => {
+    if (!EhAPI.has_cookie) {
+        throw 'Ehentai favorites RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>';
+    }
+    const favcat = ctx.params.favcat ? parseInt(ctx.params.favcat) : 0;
+    const page = ctx.params.page ? parseInt(ctx.params.page) : 0;
+    const routeParams = new URLSearchParams(ctx.params.routeParams);
+    const bittorrent = routeParams.get('bittorrent') || false;
+    const embed_thumb = routeParams.get('embed_thumb') || false;
+    const inline_set = ctx.params.order === 'posted' ? 'fs_p' : 'fs_f';
+    const items = await EhAPI.getFavoritesItems(ctx.cache, page, favcat, inline_set, bittorrent, embed_thumb);
+
+    if (EhAPI.from_ex) {
+        ctx.state.data = {
+            title: 'ExHentai Favorites',
+            link: `https://exhentai.org/favorites.php?favcat=${favcat}&inline_set=${inline_set}`,
+            item: items,
+        };
+    } else {
+        ctx.state.data = {
+            title: 'E-Hentai Favorites',
+            link: `https://e-hentai.org/favorites.php?favcat=${favcat}&inline_set=${inline_set}`,
+            item: items,
+        };
+    }
+};

package/lib/v2/ehentai/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/favorites/:favcat?/:order?/:page?/:routeParams?': ['yindaheng98'],
+    '/search/:params?/:page?/:routeParams?': ['yindaheng98'],
+    '/tag/:tag/:page?/:routeParams?': ['yindaheng98'],
+};

package/lib/v2/ehentai/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/favorites/:favcat?/:order?/:page?/:routeParams?', require('./favorites'));
+    router.get('/search/:params?/:page?/:routeParams?', require('./search'));
+    router.get('/tag/:tag/:page?/:routeParams?', require('./tag'));
+};

package/lib/v2/ehentai/search.js

@@ -0,0 +1,36 @@
+const EhAPI = require('./ehapi');
+
+module.exports = async (ctx) => {
+    const page = ctx.params.page;
+    let params = ctx.params.params;
+    const routeParams = new URLSearchParams(ctx.params.routeParams);
+    const bittorrent = routeParams.get('bittorrent') || false;
+    const embed_thumb = routeParams.get('embed_thumb') || false;
+    let items;
+    if (page) {
+        // 如果定义了page，就要覆盖params
+        params = params.replace(/&*page=[^&]$/, '').replace(/page=[^&]&/, '');
+        items = await EhAPI.getSearchItems(ctx.cache, params, parseInt(page), bittorrent, embed_thumb);
+    } else {
+        items = await EhAPI.getSearchItems(ctx.cache, params, undefined, bittorrent, embed_thumb);
+    }
+    let title = params;
+    const match = /f_search=([^&]+)/.exec(title);
+    if (match !== null) {
+        title = match[1];
+    }
+
+    if (EhAPI.from_ex) {
+        ctx.state.data = {
+            title: title + ' - ExHentai Search ',
+            link: `https://exhentai.org/?${params}`,
+            item: items,
+        };
+    } else {
+        ctx.state.data = {
+            title: title + ' - E-Hentai Search ',
+            link: `https://e-hentai.org/?${params}`,
+            item: items,
+        };
+    }
+};

package/lib/v2/ehentai/tag.js

@@ -0,0 +1,24 @@
+const EhAPI = require('./ehapi');
+
+module.exports = async (ctx) => {
+    const page = ctx.params.page ? parseInt(ctx.params.page) : 0;
+    const tag = ctx.params.tag;
+    const routeParams = new URLSearchParams(ctx.params.routeParams);
+    const bittorrent = routeParams.get('bittorrent') || false;
+    const embed_thumb = routeParams.get('embed_thumb') || false;
+    const items = await EhAPI.getTagItems(ctx.cache, tag, page, bittorrent, embed_thumb);
+
+    if (EhAPI.from_ex) {
+        ctx.state.data = {
+            title: tag + ' - ExHentai Tag',
+            link: `https://exhentai.org/tag/${tag}`,
+            item: items,
+        };
+    } else {
+        ctx.state.data = {
+            title: tag + ' - E-Hentai Tag',
+            link: `https://e-hentai.org/tag/${tag}`,
+            item: items,
+        };
+    }
+};

package/lib/v2/elasticsearch-cn/index.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate, parseRelativeDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const params = ctx.params.params ?? '';
+
+    const rootUrl = 'https://elasticsearch.cn';
+    const currentUrl = `${rootUrl}${params ? `/${params}` : ''}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.aw-question-content')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const a = item.find('h4 a');
+            const pubDate = item.find('span.text-color-999').not('.pull-right').first().text().split('•').pop().trim();
+
+            return {
+                title: a.text(),
+                link: a.attr('href'),
+                author: item.find('.aw-user-name').text(),
+                pubDate: timezone(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test(pubDate) ? parseDate(pubDate) : parseRelativeDate(pubDate), +8),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('.markitup-box').first().html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/elasticsearch-cn/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:params?': ['nczitzk'],
+};

package/lib/v2/elasticsearch-cn/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'elasticsearch.cn': {
+        _name: 'Elastic 中文社区',
+        '.': [
+            {
+                title: '发现',
+                docs: 'https://docs.rsshub.app/bbs.html#elastic-zhong-wen-she-qu-fa-xian',
+                source: ['/:params', '/'],
+                target: '/elasticsearch-cn/:params',
+            },
+        ],
+    },
+};

package/lib/v2/elasticsearch-cn/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:params?', require('./index'));
+};

package/lib/v2/elsevier/issue.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const journal = ctx.params.journal;
+    const issue = 'Volume ' + ctx.params.issue.replace('-', ' Issue ');
+    const host = 'https://www.sciencedirect.com';
+    const issueUrl = `${host}/journal/${journal}/vol/${ctx.params.issue.replace('-', '/issue/')}`;
+
+    const response = await got(issueUrl, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+    const jrnlName = $('.anchor.js-title-link').text();
+    const list = $('.js-article')
+        .map((_, item) => {
+            const title = $(item).find('.js-article-title').text();
+            const authors = $(item).find('.js-article__item__authors').text();
+            const link = $(item).find('.article-content-title').attr('href');
+            const id = $(item).find('.article-content-title').attr('id');
+            return {
+                title,
+                link,
+                id,
+                authors,
+                issue,
+            };
+        })
+        .get();
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response2 = await got(`${host}/science/article/pii/${item.id}`, {
+                    cookieJar,
+                });
+
+                const $2 = cheerio.load(response2.data);
+                $2('.section-title').remove();
+                item.doi = $2('.doi').attr('href').replace('https://doi.org/', '');
+                item.abstract = $2('.abstract.author').text();
+                item.description = renderDesc(item);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${jrnlName} - ${issue}`,
+        link: issueUrl,
+        item: items,
+    };
+};

package/lib/v2/elsevier/journal.js

@@ -0,0 +1,72 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const journal = ctx.params.journal;
+    const host = 'https://www.sciencedirect.com';
+    const jrnlUrl = `${host}/journal/${journal}`;
+
+    const response = await got(jrnlUrl, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+    const jrnlName = $('.anchor.js-title-link').text();
+    const issueUrl = `${host}${$('.anchor.js-volume').attr('href')}`;
+    let issue = '';
+    if (issueUrl.match('suppl') !== null) {
+        issue = 'Volume ' + issueUrl.match('vol/(.*)/suppl')[1];
+    } else if (issueUrl.match('issue') !== null) {
+        issue = 'Volume ' + issueUrl.match('vol/(.*)/issue')[1] + ' Issue ' + issueUrl.match('/issue/(.*)')[1];
+    }
+
+    const response2 = await got(issueUrl, {
+        cookieJar,
+    });
+    const $2 = cheerio.load(response2.data);
+    const list = $2('.js-article')
+        .map((_, item) => {
+            const title = $2(item).find('.js-article-title').text();
+            const authors = $2(item).find('.js-article__item__authors').text();
+            const link = $2(item).find('.article-content-title').attr('href');
+            const id = $2(item).find('.article-content-title').attr('id');
+            return {
+                title,
+                link,
+                id,
+                authors,
+                issue,
+            };
+        })
+        .get();
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response3 = await got(`${host}/science/article/pii/${item.id}`, {
+                    cookieJar,
+                });
+                const $3 = cheerio.load(response3.data);
+                $3('.section-title').remove();
+                item.doi = $3('.doi').attr('href').replace('https://doi.org/', '');
+                item.abstract = $3('.abstract.author').text();
+                item.description = renderDesc(item);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: jrnlName,
+        link: jrnlUrl,
+        item: items,
+    };
+};

package/lib/v2/elsevier/maintainer.js

@@ -1,4 +1,4 @@
 module.exports = {
-    '/:journal/latest': ['Derekmini', 'sunwolf-swb'],
-    '/:journal/vol/:id': ['Derekmini', 'sunwolf-swb'],
+    '/:journal': ['Derekmini', 'sunwolf-swb'],
+    '/:journal/:issue': ['Derekmini', 'sunwolf-swb'],
 };

package/lib/v2/elsevier/radar.js

@@ -3,16 +3,16 @@ module.exports = {
         _name: 'Elsevier',
         www: [
             {
-                title: 'latest',
+                title: 'Journal',
                 docs: 'https://docs.rsshub.app/journal.html#Elsevier',
                 source: '/journal/:journal/*',
-                target: '/elsevier/:journal/latest',
+                target: '/elsevier/:journal',
             },
             {
-                title: 'volume',
+                title: 'Issue',
                 docs: 'https://docs.rsshub.app/journal.html#Elsevier',
-                source: '/journal/:journal/vol/:id',
-                target: '/elsevier/:journal/vol/:id',
+                source: '/journal/:journal/vol/:issue',
+                target: '/elsevier/:journal/:issue',
             },
         ],
     },

package/lib/v2/elsevier/router.js

@@ -1,4 +1,6 @@
 module.exports = function (router) {
-    router.get('/:journal/latest', require('./latest'));
-    router.get('/:journal/vol/:id', require('./volume'));
+    router.get('/:journal/latest', require('./journal')); // deprecated
+    router.get('/:journal/vol/:issue', require('./issue')); // deprecated
+    router.get('/:journal', require('./journal'));
+    router.get('/:journal/:issue', require('./issue'));
 };

package/lib/v2/elsevier/templates/description.art

@@ -1,9 +1,11 @@
 <p>
-    <span><big>{{ item.title }}</big></span><br />
-    <span><small><i>{{ item.authors }}</i></small></span><br />
-    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br />
-    <span><small><i>{{ item.volume }}</i></small></span><br />
+    <span><big>{{ item.title }}</big></span><br>
 </p>
 <p>
-    <span>{{ item.abstract }}</span><br />    
+    <span><small><i>{{ item.authors }}</i></small></span><br>
+    <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br>
+    <span><small><i>{{ item.issue }}</i></small></span><br>
+</p>
+<p>
+    <span>{{ item.abstract }}</span><br>
 </p>

package/lib/v2/embassy/index.js

@@ -0,0 +1,60 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const supportedList = require('./supportedList');
+
+module.exports = async (ctx) => {
+    const country = ctx.params.country;
+    const city = ctx.params.city ?? undefined;
+
+    let config = supportedList[country.toLowerCase()];
+    let desc;
+
+    if (city === undefined) {
+        desc = `中国驻${config.countryCN}大使馆 -- 重要通知`;
+    } else {
+        config = config.consulates[city.toLowerCase()];
+        desc = `中国驻${config.cityCN}领事馆 -- 重要通知`;
+    }
+
+    const link = config.link;
+    const hostname = new URL(link).hostname;
+
+    const res = await got(link);
+    const $ = cheerio.load(res.data);
+
+    const list = [];
+
+    $(config.list)
+        .slice(0, 10)
+        .each((i, e) => {
+            const temp = new URL($(e).attr('href'), link);
+            if (temp.hostname === hostname) {
+                list.push(temp);
+            }
+        });
+
+    const out = await Promise.all(
+        list.map((link) =>
+            ctx.cache.tryGet(link.href, async () => {
+                const response = await got(link);
+                const $ = cheerio.load(response.data);
+                const single = {
+                    title: $(config.title).text(),
+                    link,
+                    description: $(config.description).html(),
+                    pubDate: parseDate($(config.pubDate).text().replace('(', '').replace(')', '')),
+                };
+                return single;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: desc,
+        description: desc,
+        link,
+        item: out,
+    };
+};

package/lib/v2/embassy/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:country/:city?': ['HenryQW'],
+};