rsshub 1.0.0-master.f961961 → 1.0.0-master.f992838

package/lib/v2/dmzj/radar.js

@@ -0,0 +1,67 @@
+module.exports = {
+    'dmzj.com': {
+        _name: '动漫之家',
+        news: [
+            {
+                title: '宅新闻',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/',
+                target: '/dmzj/news',
+            },
+            {
+                title: '漫画情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manhuaqingbao',
+                target: '/dmzj/news/manhuaqingbao',
+            },
+            {
+                title: '轻小说情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/qingxiaoshuoqingbao',
+                target: '/dmzj/news/qingxiaoshuoqingbao',
+            },
+            {
+                title: '动漫周边',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manhuazhoubian',
+                target: '/dmzj/news/manhuazhoubian',
+            },
+            {
+                title: '声优情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/shengyouqingbao',
+                target: '/dmzj/news/shengyouqingbao',
+            },
+            {
+                title: '音乐资讯',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/yinyuezixun',
+                target: '/dmzj/news/yinyuezixun',
+            },
+            {
+                title: '游戏资讯',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/youxizixun',
+                target: '/dmzj/news/youxizixun',
+            },
+            {
+                title: '美图欣赏',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/meituxinshang',
+                target: '/dmzj/news/meituxinshang',
+            },
+            {
+                title: '漫展情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manzhanqingbao',
+                target: '/dmzj/news/manzhanqingbao',
+            },
+            {
+                title: '大杂烩',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/dazahui',
+                target: '/dmzj/news/dazahui',
+            },
+        ],
+    },
+};

package/lib/v2/dmzj/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/news/:category?', require('./news'));
+};

package/lib/v2/douban/other/weekly_best.js

@@ -1,4 +1,6 @@
 const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
     const type = ctx.params.type || 'movie_weekly_best';
@@ -28,17 +30,22 @@ module.exports = async (ctx) => {
         link: `https://m.douban.com/subject_collection/${type}`,
         description: infoResponse.data.description,
 
-        item: data.map(({ title, info, cover, cover_url, url, rating, year, release_date, null_rating_reason, description }) => {
-            const release = `${year}.${release_date}`;
+        item: data.map(({ title, cover, cover_url, url, rating, null_rating_reason, description, card_subtitle, photos }) => {
             const rate = rating ? `${rating.value.toFixed(1)}分` : null_rating_reason;
             if (cover && cover.url) {
                 cover_url = cover.url;
             }
-            description = `标题：${title} <br> 影片信息：${info} <br> 上映日期：${release} <br> 评分：${rate} <br> <img src="${cover_url}"> <p>${description}</p>`;
 
             return {
                 title,
-                description,
+                description: art(path.join(__dirname, '../templates/weekly_best.art'), {
+                    title,
+                    card_subtitle,
+                    description,
+                    rate,
+                    cover_url,
+                    photos,
+                }),
                 link: url,
             };
         }),

package/lib/v2/douban/templates/weekly_best.art

@@ -0,0 +1,10 @@
+<p>标题：{{title}}</p>
+<p>评分：{{rate}}</p>
+<p>标签：{{card_subtitle}}</p>
+<p>影片信息：{{description}}</p>
+<p>
+    <img src="{{cover_url}}" />
+    {{each photos}}
+        <img src="{{$value}}" />
+    {{/each}}
+</p>

package/lib/v2/douyin/hashtag.js

@@ -1,4 +1,3 @@
-const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const config = require('@/config').value;
@@ -23,19 +22,26 @@ module.exports = async (ctx) => {
             const browser = await require('@/utils/puppeteer')();
             const page = await browser.newPage();
             await page.setRequestInterception(true);
+            let awemeList = '';
             page.on('request', (request) => {
-                request.resourceType() === 'document' ? request.continue() : request.abort();
+                request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+            });
+            page.on('response', async (response) => {
+                const request = response.request();
+                if (request.url().includes('/v1/web/challenge/aweme')) {
+                    awemeList = await response.json();
+                }
             });
             await page.goto(tagUrl, {
-                waitUntil: 'domcontentloaded',
+                waitUntil: 'networkidle2',
             });
             await page.waitForSelector('#RENDER_DATA');
-            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            const html = await page.evaluate(() => document.getElementById('RENDER_DATA').innerText);
             await browser.close();
 
-            const $ = cheerio.load(html);
-            const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').text()));
-            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail && renderData[key].defaultData);
+            const renderData = JSON.parse(decodeURIComponent(html));
+            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail);
+            renderData[dataKey].defaultData = awemeList.aweme_list;
             return renderData[dataKey];
         },
         config.cache.routeExpire,
@@ -49,7 +55,7 @@ module.exports = async (ctx) => {
     const posts = tagData.defaultData;
     const items = posts.map((post) => {
         // parse video
-        let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
+        let videoList = post.video && post.video.bit_rate && post.video.bit_rate.map((bit_rate) => resolveUrl(bit_rate.play_addr.url_list[0]));
         if (relay) {
             videoList = videoList.map((item) => proxyVideo(item, relay));
         }
@@ -57,14 +63,9 @@ module.exports = async (ctx) => {
         duration = duration && duration / 1000;
         let img;
         // if (!embed) {
-        //     img = post.video && post.video.dynamicCover; // dynamic cover (webp)
+        //     img = post.video && post.video.dynamic_cover && post.video.dynamic_cover.url_list[post.video.dynamic_cover.url_list.length - 1]; // dynamic cover (webp)
         // }
-        img =
-            img ||
-            (post.video &&
-                ((post.video.coverUrlList && post.video.coverUrlList[post.video.coverUrlList.length - 1]) || // HD
-                    post.video.originCover || // LD
-                    post.video.cover)); // even more LD
+        img = img || (post.video && post.video.origin_cover && post.video.origin_cover.url_list[post.video.origin_cover.url_list.length - 1]);
         img = img && resolveUrl(img);
 
         // render description
@@ -76,10 +77,10 @@ module.exports = async (ctx) => {
         return {
             title: post.desc,
             description,
-            link: `https://www.douyin.com/video/${post.awemeId}`,
-            pubDate: parseDate(post.createTime * 1000),
-            category: post.textExtra.map((extra) => extra.hashtagName),
-            author: post.authorInfo.nickname,
+            link: `https://www.douyin.com/video/${post.aweme_id}`,
+            pubDate: parseDate(post.create_time, 'X'),
+            category: post.text_extra.map((extra) => extra.hashtag_name),
+            author: post.author.nickname,
         };
     });
 

package/lib/v2/douyin/live.js

@@ -1,5 +1,5 @@
 const config = require('@/config').value;
-const { getOriginAvatar } = require('./utils');
+const { getOriginAvatar, universalGet } = require('./utils');
 
 module.exports = async (ctx) => {
     const { rid } = ctx.params;
@@ -9,26 +9,7 @@ module.exports = async (ctx) => {
 
     const pageUrl = `https://live.douyin.com/${rid}`;
 
-    const renderData = await ctx.cache.tryGet(
-        `douyin:live:${rid}`,
-        async () => {
-            const browser = await require('@/utils/puppeteer')();
-            const page = await browser.newPage();
-            await page.setRequestInterception(true);
-            page.on('request', (request) => {
-                request.resourceType() === 'document' ? request.continue() : request.abort();
-            });
-            await page.goto(pageUrl, {
-                waitUntil: 'domcontentloaded',
-            });
-            const data = JSON.parse(decodeURIComponent(await page.evaluate(() => document.querySelector('#RENDER_DATA').innerText)));
-            browser.close();
-
-            return data;
-        },
-        config.cache.routeExpire,
-        false
-    );
+    const renderData = await ctx.cache.tryGet(`douyin:live:${rid}`, () => universalGet(pageUrl, 'live', ctx), config.cache.routeExpire, false);
 
     if (renderData[Object.keys(renderData).find((k) => renderData[k].status_code)].status_code !== '0') {
         throw `Status code ${renderData[Object.keys(renderData).find((k) => renderData[k].status_code)].status_code}`;

package/lib/v2/douyin/user.js

@@ -1,9 +1,8 @@
-const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const config = require('@/config').value;
 const { fallback, queryToBoolean } = require('@/utils/readable-social');
-const { templates, resolveUrl, proxyVideo, getOriginAvatar } = require('./utils');
+const { templates, resolveUrl, proxyVideo, getOriginAvatar, universalGet } = require('./utils');
 
 module.exports = async (ctx) => {
     const uid = ctx.params.uid;
@@ -20,27 +19,13 @@ module.exports = async (ctx) => {
     const pageData = await ctx.cache.tryGet(
         `douyin:user:${uid}`,
         async () => {
-            const browser = await require('@/utils/puppeteer')();
-            const page = await browser.newPage();
-            await page.setRequestInterception(true);
-            page.on('request', (request) => {
-                request.resourceType() === 'document' ? request.continue() : request.abort();
-            });
-            await page.goto(pageUrl, {
-                waitUntil: 'domcontentloaded',
-            });
-            const html = await page.evaluate(() => document.documentElement.innerHTML);
-            await browser.close();
-
-            const $ = cheerio.load(html);
-            const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
+            const renderData = await universalGet(pageUrl, 'user', ctx);
             const dataKey = Object.keys(renderData).filter((key) => renderData[key].user && renderData[key].post)[0];
             return renderData[dataKey];
         },
         config.cache.routeExpire,
         false
     );
-
     const userInfo = pageData.user.user;
     const userNickName = userInfo.nickname;
     const userDescription = userInfo.desc;

package/lib/v2/douyin/utils.js

@@ -1,4 +1,8 @@
 const path = require('path');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const logger = require('@/utils/logger');
+const config = require('@/config').value;
 
 const templates = {
     desc: path.join(__dirname, 'templates/desc.art'),
@@ -42,9 +46,90 @@ const getOriginAvatar = (url) =>
         .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
         .replace(/~\w+_\d+x\d+/g, '');
 
+const extractRenderData = (html) => {
+    if (!html) {
+        throw Error('Empty html. The request may be filtered by WAF.');
+    }
+
+    const $ = cheerio.load(html);
+    const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
+    if (!renderData) {
+        const title = $('title').text();
+        if (title.includes('验证')) {
+            throw Error(title);
+        } else if ($('#captcha_container').length > 0) {
+            throw Error('Captcha required.');
+        }
+        throw Error('Failed to get render data.');
+    } else if (renderData.isSpider) {
+        throw Error('The request was considered to be from a spider.');
+    } else {
+        return renderData;
+    }
+};
+
+const gotGet = async (url, ua) => {
+    const response = await got(url, {
+        headers: {
+            'User-Agent': ua, // magic!
+        },
+    });
+    return extractRenderData(response.data);
+};
+
+const puppeteerGet = async (pageUrl) => {
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    await page.goto(pageUrl, {
+        waitUntil: 'domcontentloaded',
+    });
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    await browser.close();
+    return extractRenderData(html);
+};
+
+const universalGet = async (url, route, ctx) => {
+    const cacheKey = `douyin:utils:universalGet:gotFirst:${route}`;
+    const gotFirst = await ctx.cache.tryGet(
+        cacheKey,
+        () => '1',
+        config.cache.contentExpire,
+        false // no refresh now
+    );
+
+    const ua = route === 'live' ? 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' : 'curl/7.86.0';
+
+    let data;
+    if (gotFirst === '1') {
+        try {
+            data = await gotGet(url, ua);
+            ctx.cache.set(cacheKey, '1', config.cache.contentExpire); // refresh if success
+        } catch (e) {
+            logger.warn('Failed to get data with got, trying puppeteer: ' + e.message);
+            data = await puppeteerGet(url);
+            ctx.cache.set(cacheKey, '0', config.cache.contentExpire); // only set if success
+        }
+    } else {
+        try {
+            data = await puppeteerGet(url);
+            ctx.cache.set(cacheKey, '0', config.cache.contentExpire); // refresh if success
+        } catch (e) {
+            logger.warn('Failed to get data with puppeteer, trying got: ' + e.message);
+            data = await gotGet(url, ua);
+            ctx.cache.set(cacheKey, '1', config.cache.contentExpire); // only set if success
+        }
+    }
+    return data;
+};
+
 module.exports = {
     templates,
     resolveUrl,
     proxyVideo,
     getOriginAvatar,
+    universalGet,
 };

package/lib/v2/dut/index.js

@@ -3,9 +3,13 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const defaults = require('./defaults');
 const shortcuts = require('./shortcuts');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const site = ctx.params[0] ?? 'news';
+    if (!isValidHost(site)) {
+        throw Error('Invalid site');
+    }
 
     let items;
     let category = ctx.params[1] ?? (defaults.hasOwnProperty(site) ? defaults[site] : '');

package/lib/v2/eagle/blog.js

@@ -1,16 +1,20 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-
+const { isValidHost } = require('@/utils/valid-host');
 const cateList = ['all', 'design-resources', 'learn-design', 'inside-eagle'];
 
 module.exports = async (ctx) => {
     let cate = ctx.params.cate ?? 'all';
     let language = ctx.params.language ?? 'cn';
-    if (cateList.indexOf(cate) === -1) {
+    if (!isValidHost(cate) || !isValidHost(language)) {
+        throw Error('Invalid host');
+    }
+    if (!cateList.includes(cate)) {
         language = cate;
         cate = 'all';
     }
+
     const host = `https://${language}.eagle.cool`;
     const url = `${host}/blog/${cate === 'all' ? '' : cate}`;
 

package/lib/v2/economist/espresso.js

@@ -15,36 +15,66 @@ module.exports = async (ctx) => {
         false
     );
 
-    const gobbets = $('._gobbets ._gobbet p')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-            return {
-                title: item.text(),
-                description: item.html(),
-                link,
-                guid: item.text(),
-            };
-        });
-
-    const articles = $('._articles ._article')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-            const title = item.find('h3').text();
-            item.find('h3').remove();
-            return {
-                title,
-                description: item.html(),
-                link,
-                guid: item.text(),
-            };
-        });
+    const nextData = JSON.parse($('script#__NEXT_DATA__').text());
+    const parts = nextData.props.pageProps.content.hasPart.parts[0].hasPart.parts.filter((part) => part.type.includes('Article'));
+
+    const renderHTML = (node) => {
+        let el;
+        switch (node.type) {
+            case 'tag':
+                el = $(`<${node.name}>`);
+                for (const name in node.attribs) {
+                    el.attr(name, node.attribs[name]);
+                }
+                if (node.children) {
+                    el.append(node.children.map(renderHTML));
+                }
+                return el;
+            case 'text':
+                return node.data;
+            default:
+                el = $('<div>');
+                el.append(node.map(renderHTML));
+                return el;
+        }
+    };
+
+    const gobbetPart = parts.shift();
+    const gobbets = gobbetPart.text.map((node, index) => {
+        const html = renderHTML(node).html();
+        return {
+            guid: `${gobbetPart.id}/${index}`,
+            link,
+            title: gobbetPart.headline,
+            pubDate: gobbetPart.datePublished,
+            description: html,
+        };
+    });
+
+    const articles = parts.map((part) => {
+        const image = part.image.main;
+        const imgNode = {
+            type: 'tag',
+            name: 'img',
+            attribs: { width: image.width, height: image.height, src: image.url.canonical },
+        };
+
+        const html = renderHTML([imgNode, ...part.text]).html();
+
+        return {
+            link,
+            guid: part.id,
+            title: part.headline,
+            pubDate: part.datePublished,
+            description: html,
+        };
+    });
 
     ctx.state.data = {
         title: $('head title').text(),
         link,
         description: $('meta[property="og:description"]').attr('content'),
+        language: 'en-gb',
         item: [...gobbets, ...articles],
     };
 };

package/lib/v2/economist/global-business-review.js

@@ -0,0 +1,86 @@
+const got = require('@/utils/got');
+
+const ALLOW_LANGUAGE = {
+    en: 'en_GB',
+    cn: 'zh_CN',
+    tw: 'zh_TW',
+};
+
+const TITLE = {
+    en: 'The Economist Global Business Review',
+    cn: '经济学人 · 商论',
+    tw: '經濟學人 · 商論',
+};
+
+const DESCRIPTION = {
+    en: 'The Economist Global Business Review is a new bilingual digital app from the editors of The Economist Group.',
+    cn: '《经济学人·商论》是《经济学人》2015年5月推出的旗下中英双语APP，萃取《经济学人》在商业、金融、科技等领域的精华文章，为中国读者呈现全球视角的深度分析，并鼓励中国的读者批判性地思考中国和全球重大议题。',
+    tw: '《經濟學人·商論》是經濟學人集團官方中英雙語電子APP，萃取《經濟學人》在商業、金融、科技等領域的精華文章，為中國讀者呈現全球視角的深度分析，並鼓勵中國的讀者批判性地思考中國和全球的重大議題。',
+};
+
+const parseContent = function (item, article_id, language) {
+    if (item.type === 'paragraph') {
+        return parseParagraph(item.data, language);
+    } else if (item.type === 'image') {
+        return parseImage(item.data, article_id, language);
+    } else if (item.type === 'subtitle') {
+        return parseParagraph(item.data, language, 'h3');
+    }
+};
+
+const parseTitle = function (data, language) {
+    const content = Object.assign({}, ...data.map((x) => ({ [x.lang]: x.text })));
+    return language.map((item) => content[ALLOW_LANGUAGE[item]]).join('');
+};
+
+const parseParagraph = function (data, language, type = 'p') {
+    const content = Object.assign({}, ...data.map((x) => ({ [x.lang]: x.text })));
+    return `<div>${language.map((item) => `<${type}>${content[ALLOW_LANGUAGE[item]]}</${type}>`).join('')}</div>`;
+};
+
+const parseImage = function (data, article_id, language) {
+    const content = Object.assign({}, ...data.map((x) => ({ [x.lang]: x.image_path })));
+    return `<div><img src="https://businessreviewglobal-cdn.com/article_images/${article_id}/${encodeURIComponent(content[ALLOW_LANGUAGE[language[0]]])}"/></div>`;
+};
+
+const getArticleDetail = (article_id, language, ctx) => {
+    const link = `https://api.hummingbird.businessreview.global/api/article/index?id=${article_id}`;
+    return ctx.cache.tryGet(`${link}:${language.join('-')}`, async () => {
+        const response = await got({
+            method: 'get',
+            url: link,
+        });
+        const data = response.data.body;
+        let content = '';
+        content += parseParagraph(data.rubric, language);
+        content += data.content.map((item) => parseContent(item, article_id, language)).join('');
+        return content;
+    });
+};
+
+module.exports = async (ctx) => {
+    const language = (ctx.params.language && ctx.params.language.split('-').filter((item, index, arr) => ALLOW_LANGUAGE[item] && arr.indexOf(item, 0) === index)) || ['cn', 'en'];
+    const main_language = language[0];
+
+    const response = await got({
+        method: 'get',
+        url: 'https://api.hummingbird.businessreview.global/api/toc/get_articles',
+    });
+
+    const items = await Promise.all(
+        response.data.articles.new.slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 10).map(async (item) => ({
+            title: parseTitle(item.body.title, [main_language]),
+            description: await getArticleDetail(item.article_id, language, ctx),
+            category: parseTitle(item.body.fly_title, [main_language]),
+            link: `https://www.businessreview.global/latest/${item.article_id}`,
+            pubDate: item.publication_date,
+        }))
+    );
+
+    ctx.state.data = {
+        title: TITLE[main_language],
+        link: 'https://www.businessreview.global',
+        description: DESCRIPTION[main_language],
+        item: items,
+    };
+};

package/lib/v2/economist/maintainer.js

@@ -2,5 +2,6 @@ module.exports = {
     '/download': ['nczitzk'],
     '/espresso': ['TonyRL'],
     '/gre-vocabulary': ['xyqfer'],
+    '/global-business-review/:language?': ['prnake'],
     '/:endpoint': ['ImSingee'],
 };

package/lib/v2/economist/radar.js

@@ -24,4 +24,15 @@ module.exports = {
             },
         ],
     },
+    'businessreview.global': {
+        _name: 'The Economist',
+        '.': [
+            {
+                title: '商论',
+                docs: 'https://docs.rsshub.app/traditional-media.html#the-economist',
+                source: ['/'],
+                target: '/economist/global-business-review',
+            },
+        ],
+    },
 };

package/lib/v2/economist/router.js

@@ -2,5 +2,6 @@ module.exports = (router) => {
     router.get('/download', require('./download'));
     router.get('/espresso', require('./espresso'));
     router.get('/gre-vocabulary', require('./gre-vocabulary'));
+    router.get('/global-business-review/:language?', require('./global-business-review'));
     router.get('/:endpoint', require('./full'));
 };