rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/lib/v2/domp4/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'domp4.cc': {
+        _name: 'domp4电影',
+        '.': [
+            {
+                title: '最近更新',
+                docs: 'https://docs.rsshub.app/multimedia.html#domp4-ying-shi',
+                source: ['/', '/custom/update.html'],
+                target: '/domp4/latest/:type?',
+            },
+            {
+                title: '剧集订阅',
+                docs: 'https://docs.rsshub.app/multimedia.html#domp4-ying-shi',
+                source: '/html/:id',
+                target: '/domp4/detail/:id',
+            },
+            {
+                title: '剧集订阅',
+                docs: 'https://docs.rsshub.app/multimedia.html#domp4-ying-shi',
+                source: '/detail/:id',
+                target: '/domp4/detail/:id',
+            },
+        ],
+    },
+};

package/lib/v2/domp4/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/detail/:id', require('./detail'));
+    router.get('/latest/:type?', require('./latest'));
+};

package/lib/v2/domp4/utils.js

@@ -0,0 +1,72 @@
+/**
+ * trackers from https://www.domp4.cc/Style/2020/js/base.js?v=2
+ */
+const magnetTrackers = [
+    'https://tracker.iriseden.fr:443/announce',
+    'https://tr.highstar.shop:443/announce',
+    'https://tr.fuckbitcoin.xyz:443/announce',
+    'https://tr.doogh.club:443/announce',
+    'https://tr.burnabyhighstar.com:443/announce',
+    'https://t.btcland.xyz:443/announce',
+    'http://vps02.net.orel.ru:80/announce',
+    'https://tracker.kuroy.me:443/announce',
+    'http://tr.cili001.com:8070/announce',
+    'http://t.overflow.biz:6969/announce',
+    'http://t.nyaatracker.com:80/announce',
+    'http://open.acgnxtracker.com:80/announce',
+    'http://nyaa.tracker.wf:7777/announce',
+    'http://home.yxgz.vip:6969/announce',
+    'http://buny.uk:6969/announce',
+    'https://tracker.tamersunion.org:443/announce',
+    'https://tracker.nanoha.org:443/announce',
+    'https://tracker.loligirl.cn:443/announce',
+    'udp://bubu.mapfactor.com:6969/announce',
+    'http://share.camoe.cn:8080/announce',
+    'udp://movies.zsw.ca:6969/announce',
+    'udp://ipv4.tracker.harry.lu:80/announce',
+    'udp://tracker.sylphix.com:6969/announce',
+    'http://95.216.22.207:9001/announce',
+];
+
+/**
+ * compose magnet url with trackers
+ */
+function composeMagnetUrl(magnet, trackers = magnetTrackers) {
+    return `${magnet}&tr=${trackers.join('&tr=')}`;
+}
+
+/**
+ * method for generating magnet url
+ * from the detail page source
+ */
+function decodeCipherText(p, a, c, k, e, d) {
+    e = function (c) {
+        return (c < a ? '' : e(parseInt(c / a))) + ((c = c % a) > 35 ? String.fromCharCode(c + 29) : c.toString(36));
+    };
+    if (!''.replace(/^/, String)) {
+        while (c--) {
+            d[e(c)] = k[c] || e(c);
+        }
+        k = [
+            function (e) {
+                return d[e];
+            },
+        ];
+        e = function () {
+            return '\\w+';
+        };
+        c = 1;
+    }
+    while (c--) {
+        if (k[c]) {
+            p = p.replace(new RegExp('\\b' + e(c) + '\\b', 'g'), k[c]);
+        }
+    }
+    return p;
+}
+
+module.exports = {
+    magnetTrackers,
+    composeMagnetUrl,
+    decodeCipherText,
+};

package/lib/v2/douyin/hashtag.js

@@ -0,0 +1,133 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const config = require('@/config').value;
+const { fallback, queryToBoolean } = require('@/utils/readable-social');
+
+const templates = {
+    desc: path.join(__dirname, 'templates/desc.art'),
+    cover: path.join(__dirname, 'templates/cover.art'),
+    embed: path.join(__dirname, 'templates/embed.art'),
+    iframe: path.join(__dirname, 'templates/iframe.art'),
+};
+
+const resolveUrl = (url, tls = true, forceResolve = false) => {
+    if (!url) {
+        return '';
+    }
+    if (url.startsWith('//')) {
+        return (tls ? 'https:' : 'http:') + url;
+    }
+    if (forceResolve && !url.match(/^https?:\/\//)) {
+        return (tls ? 'https://' : 'http://') + url;
+    }
+    return url;
+};
+
+const proxyVideo = (url, proxy) => {
+    if (!(url && proxy)) {
+        return url + '';
+    }
+    if (proxy.includes('?')) {
+        if (!proxy.endsWith('=')) {
+            proxy += '=';
+        }
+        return proxy + encodeURIComponent(url);
+    } else {
+        if (!proxy.endsWith('/')) {
+            proxy += '/';
+        }
+        return proxy + url;
+    }
+};
+
+const getOriginAvatar = (url) =>
+    resolveUrl(url)
+        .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
+        .replace(/~\w+_\d+x\d+/g, '');
+
+module.exports = async (ctx) => {
+    const { cid } = ctx.params;
+    if (isNaN(cid)) {
+        throw Error('Invalid tag ID. Tag ID should be a number.');
+    }
+    const routeParams = Object.fromEntries(new URLSearchParams(ctx.params.routeParams));
+    const embed = fallback(undefined, queryToBoolean(routeParams.embed), false); // embed video
+    const iframe = fallback(undefined, queryToBoolean(routeParams.iframe), false); // embed video in iframe
+    const relay = resolveUrl(routeParams.relay, true, true); // embed video behind a reverse proxy
+
+    const tagUrl = `https://www.douyin.com/hashtag/${cid}`;
+
+    const tagData = await ctx.cache.tryGet(
+        `douyin:hashtag:${cid}`,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' ? request.continue() : request.abort();
+            });
+            await page.goto(tagUrl, {
+                waitUntil: 'domcontentloaded',
+            });
+            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            await browser.close();
+
+            const $ = cheerio.load(html);
+            const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').text()));
+            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail && renderData[key].defaultData);
+            return renderData[dataKey];
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const tagInfo = tagData.topicDetail;
+    const tagName = tagInfo.chaName;
+    const userAvatar = getOriginAvatar(tagInfo.hashtagProfile);
+
+    const posts = tagData.defaultData;
+    const items = posts.map((post) => {
+        // parse video
+        let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
+        if (relay) {
+            videoList = videoList.map((item) => proxyVideo(item, relay));
+        }
+        let duration = post.video && post.video.duration;
+        duration = duration && duration / 1000;
+        let img;
+        // if (!embed) {
+        //     img = post.video && post.video.dynamicCover; // dynamic cover (webp)
+        // }
+        img =
+            img ||
+            (post.video &&
+                ((post.video.coverUrlList && post.video.coverUrlList[post.video.coverUrlList.length - 1]) || // HD
+                    post.video.originCover || // LD
+                    post.video.cover)); // even more LD
+        img = img && resolveUrl(img);
+
+        // render description
+        const desc = post.desc && post.desc.replace(/\n/g, '<br>');
+        let media = art(embed && videoList ? templates.embed : templates.cover, { img, videoList, duration });
+        media = embed && videoList && iframe ? art(templates.iframe, { content: media }) : media; // warp in iframe
+        const description = art(templates.desc, { desc, media });
+
+        return {
+            title: post.desc,
+            description,
+            link: `https://www.douyin.com/video/${post.awemeId}`,
+            pubDate: parseDate(post.createTime * 1000),
+            category: post.textExtra.map((extra) => extra.hashtagName),
+        };
+    });
+
+    ctx.state.data = {
+        title: tagName,
+        description: `${tagInfo.viewCount} 次播放`,
+        image: userAvatar,
+        link: tagUrl,
+        item: items,
+    };
+};

package/lib/v2/douyin/radar.js

@@ -2,6 +2,12 @@ module.exports = {
     'douyin.com': {
         _name: '抖音',
         '.': [
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/social-media.html#dou-yin',
+                source: '/hashtag/:cid',
+                target: '/douyin/hashtag/:cid',
+            },
             {
                 title: '博主',
                 docs: 'https://docs.rsshub.app/social-media.html#dou-yin',

package/lib/v2/douyin/router.js

@@ -1,3 +1,4 @@
 module.exports = function (router) {
+    router.get('/hashtag/:cid/:routeParams?', require('./hashtag'));
     router.get('/user/:uid/:routeParams?', require('./user'));
 };

package/lib/v2/douyin/user.js

@@ -49,6 +49,9 @@ const getOriginAvatar = (url) =>
 
 module.exports = async (ctx) => {
     const uid = ctx.params.uid;
+    if (!uid.startsWith('MS4wLjABAAAA')) {
+        throw Error('Invalid UID. UID should start with <b>MS4wLjABAAAA</b>.');
+    }
     const routeParams = Object.fromEntries(new URLSearchParams(ctx.params.routeParams));
     const embed = fallback(undefined, queryToBoolean(routeParams.embed), false); // embed video
     const iframe = fallback(undefined, queryToBoolean(routeParams.iframe), false); // embed video in iframe
@@ -61,7 +64,13 @@ module.exports = async (ctx) => {
         async () => {
             const browser = await require('@/utils/puppeteer')();
             const page = await browser.newPage();
-            await page.goto(pageUrl);
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' ? request.continue() : request.abort();
+            });
+            await page.goto(pageUrl, {
+                waitUntil: 'domcontentloaded',
+            });
             const html = await page.evaluate(() => document.documentElement.innerHTML);
             await browser.close();
 

package/lib/{routes → v2}/earthquake/ceic.js

@@ -1,4 +1,6 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
     let type = Number(ctx.params.type);
@@ -57,7 +59,7 @@ module.exports = async (ctx) => {
             return {
                 title: `${entity.LOCATION_C}发生${entity.M}级地震`,
                 link: `${baseUrl}/${NEW_DID}.html`,
-                pubDate: new Date(entity.O_TIME).toUTCString(),
+                pubDate: timezone(parseDate(entity.O_TIME, 'YYYY-MM-DD HH:mm:ss'), +8),
                 description: contentBuilder.join('<br>'),
                 guid: NEW_DID,
             };

package/lib/{routes → v2}/earthquake/index.js

@@ -1,7 +1,9 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
-    const region = ctx.params.region || 1;
+    const region = ctx.params.region ?? 1;
     const api = 'https://www.cea.gov.cn/eportal/ui?struts.portlet.mode=view&struts.portlet.action=/portlet/expressEarthquake!queryExpressEarthquakeList.action&pageId=363409&moduleId=a852ba487b534470a84a30f00e7d6670';
     const link = 'https://www.cea.gov.cn/cea/xwzx/zqsd/index.html';
     const response = await got({
@@ -18,14 +20,14 @@ module.exports = async (ctx) => {
 
     const out = data.map((item) => {
         const { id, epicenter, latitudes, longitudes, depth } = item;
-        const date = item.orig_time.slice(0, -2);
+        const date = item.orig_time;
         const num = item.num_mag;
 
         const description = `北京时间${date}，${epicenter}（纬度${latitudes}度，经度${longitudes}度）发生${num}级地震，震源深度${depth}千米`;
         return {
             title: `${epicenter}发生${num}级地震`,
             link: `https://www.cea.gov.cn/eportal/ui?struts.portlet.mode=view&struts.portlet.action=/portlet/expressEarthquake!toNewInfoView.action&pageId=366521&id=${id}`,
-            pubDate: new Date(date).toUTCString(),
+            pubDate: timezone(parseDate(date, 'YYYY-MM-DD HH:mm:ss'), +8),
             description,
         };
     });

package/lib/v2/earthquake/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/ceic/:type?': ['SettingDust'],
+    '/:region?': ['LogicJake'],
+};

package/lib/v2/earthquake/radar.js

@@ -0,0 +1,24 @@
+module.exports = {
+    'ac.cn': {
+        _name: '地震速报',
+        'www.ceic': [
+            {
+                title: '中国地震台',
+                docs: 'https://docs-rsshub.pages.dev/forecast.html#di-zhen-su-bao',
+                source: ['/speedsearch', '/'],
+                target: '/earthquake/ceic',
+            },
+        ],
+    },
+    'cea.gov.cn': {
+        _name: '地震速报',
+        www: [
+            {
+                title: '中国地震局',
+                docs: 'https://docs-rsshub.pages.dev/forecast.html#di-zhen-su-bao',
+                source: ['/cea/xwzx/zqsd/index.html', '/'],
+                target: '/earthquake',
+            },
+        ],
+    },
+};

package/lib/v2/earthquake/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/ceic/:type?', require('./ceic'));
+    router.get('/:region?', require('./index'));
+};

package/lib/v2/economist/espresso.js

@@ -0,0 +1,50 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const config = require('@/config').value;
+
+const link = 'https://www.economist.com/the-world-in-brief';
+
+module.exports = async (ctx) => {
+    const $ = await ctx.cache.tryGet(
+        link,
+        async () => {
+            const response = await got(link);
+            return cheerio.load(response.data);
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const gobbets = $('._gobbets ._gobbet p')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                description: item.html(),
+                link,
+                guid: item.text(),
+            };
+        });
+
+    const articles = $('._articles ._article')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const title = item.find('h3').text();
+            item.find('h3').remove();
+            return {
+                title,
+                description: item.html(),
+                link,
+                guid: item.text(),
+            };
+        });
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link,
+        description: $('meta[property="og:description"]').attr('content'),
+        item: [...gobbets, ...articles],
+    };
+};

package/lib/v2/economist/full.js

@@ -0,0 +1,42 @@
+const parser = require('@/utils/rss-parser');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const getArticleDetail = async (link, ctx) =>
+    await ctx.cache.tryGet(link, async () => {
+        const response = await got(link);
+        const $ = cheerio.load(response.data);
+        $('div.article-audio-player__center-tooltip').remove();
+        const nextData = JSON.parse($('head script[type="application/ld+json"]').text());
+
+        const article = $('#new-article-template > div > div.css-wvfzu8 > div.css-mticar').html() + $('#new-article-template > div > div.css-wvfzu8 > div > div > section').html();
+        const categories = nextData.keywords.map((k) => k);
+
+        return { article, categories };
+    });
+
+module.exports = async (ctx) => {
+    const endpoint = ctx.params.endpoint;
+    const feed = await parser.parseURL(`https://www.economist.com/${endpoint}/rss.xml`);
+
+    const items = await Promise.all(
+        feed.items.map(async (item) => {
+            const itemDetails = await getArticleDetail(item.link, ctx);
+            return {
+                title: item.title,
+                description: itemDetails.article,
+                link: item.link,
+                guid: item.guid,
+                pubDate: item.pubDate,
+                category: itemDetails.categories,
+            };
+        })
+    );
+
+    ctx.state.data = {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: items,
+    };
+};

package/lib/v2/economist/gre-vocabulary.js

@@ -0,0 +1,19 @@
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    const response = await got({
+        method: 'get',
+        url: 'https://gre.economist.com/api/keywords.json',
+    });
+
+    const word = response.data;
+
+    ctx.state.data = {
+        title: 'The Economist GRE Vocabulary',
+        link: 'https://gre.economist.com/gre-advice/gre-vocabulary/which-words-study/most-common-gre-vocabulary-list-organized-difficulty',
+        item: word.map((item) => ({
+            title: item,
+            description: item,
+        })),
+    };
+};

package/lib/v2/economist/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/download': ['nczitzk'],
+    '/espresso': ['TonyRL'],
+    '/gre-vocabulary': ['xyqfer'],
+    '/:endpoint': ['ImSingee'],
+};

package/lib/v2/economist/radar.js

@@ -0,0 +1,27 @@
+module.exports = {
+    'economist.com': {
+        _name: 'The Economist',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/traditional-media.html#the-economist',
+                source: ['/:endpoint'],
+                target: '/economist/:endpoint',
+            },
+            {
+                title: 'Espresso',
+                docs: 'https://docs.rsshub.app/traditional-media.html#the-economist',
+                source: ['/the-world-in-brief', '/espresso'],
+                target: '/economist/espresso',
+            },
+        ],
+        gre: [
+            {
+                title: 'GRE Vocabulary',
+                docs: 'https://docs.rsshub.app/traditional-media.html#the-economist',
+                source: ['/', '/gre-advice/gre-vocabulary/which-words-study/most-common-gre-vocabulary-list-organized-difficulty'],
+                target: '/economist/gre-vocabulary',
+            },
+        ],
+    },
+};

package/lib/v2/economist/router.js

@@ -0,0 +1,6 @@
+module.exports = (router) => {
+    router.get('/download', require('./download'));
+    router.get('/espresso', require('./espresso'));
+    router.get('/gre-vocabulary', require('./gre-vocabulary'));
+    router.get('/:endpoint', require('./full'));
+};

package/lib/v2/elasticsearch-cn/index.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate, parseRelativeDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const params = ctx.params.params ?? '';
+
+    const rootUrl = 'https://elasticsearch.cn';
+    const currentUrl = `${rootUrl}${params ? `/${params}` : ''}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.aw-question-content')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const a = item.find('h4 a');
+            const pubDate = item.find('span.text-color-999').not('.pull-right').first().text().split('•').pop().trim();
+
+            return {
+                title: a.text(),
+                link: a.attr('href'),
+                author: item.find('.aw-user-name').text(),
+                pubDate: timezone(/\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test(pubDate) ? parseDate(pubDate) : parseRelativeDate(pubDate), +8),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('.markitup-box').first().html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/elasticsearch-cn/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:params?': ['nczitzk'],
+};

package/lib/v2/elasticsearch-cn/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'elasticsearch.cn': {
+        _name: 'Elastic 中文社区',
+        '.': [
+            {
+                title: '发现',
+                docs: 'https://docs.rsshub.app/bbs.html#elastic-zhong-wen-she-qu-fa-xian',
+                source: ['/:params', '/'],
+                target: '/elasticsearch-cn/:params',
+            },
+        ],
+    },
+};

package/lib/v2/elasticsearch-cn/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:params?', require('./index'));
+};

package/lib/v2/elsevier/issue.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const { CookieJar } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+module.exports = async (ctx) => {
+    const journal = ctx.params.journal;
+    const issue = 'Volume ' + ctx.params.issue.replace('-', ' Issue ');
+    const host = 'https://www.sciencedirect.com';
+    const issueUrl = `${host}/journal/${journal}/vol/${ctx.params.issue.replace('-', '/issue/')}`;
+
+    const response = await got(issueUrl, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+    const jrnlName = $('.anchor.js-title-link').text();
+    const list = $('.js-article')
+        .map((_, item) => {
+            const title = $(item).find('.js-article-title').text();
+            const authors = $(item).find('.js-article__item__authors').text();
+            const link = $(item).find('.article-content-title').attr('href');
+            const id = $(item).find('.article-content-title').attr('id');
+            return {
+                title,
+                link,
+                id,
+                authors,
+                issue,
+            };
+        })
+        .get();
+
+    const renderDesc = (item) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        });
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response2 = await got(`${host}/science/article/pii/${item.id}`, {
+                    cookieJar,
+                });
+
+                const $2 = cheerio.load(response2.data);
+                $2('.section-title').remove();
+                item.doi = $2('.doi').attr('href').replace('https://doi.org/', '');
+                item.abstract = $2('.abstract.author').text();
+                item.description = renderDesc(item);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${jrnlName} - ${issue}`,
+        link: issueUrl,
+        item: items,
+    };
+};