rsshub 1.0.0-master.fb2a407 → 1.0.0-master.fc5a2f6

package/lib/v2/diandong/news.js

@@ -0,0 +1,50 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const rootUrl = 'https://webapi.diandong.com';
+
+const titleMap = {
+    0: '推荐',
+    29: '新车',
+    61: '导购',
+    30: '试驾',
+    75: '用车',
+    22: '技术',
+    24: '政策',
+    23: '行业',
+};
+
+module.exports = async (ctx) => {
+    const cate = ctx.params.cate ?? 0;
+    const limit = ctx.query.limit ? Number(ctx.query.limit) : 25;
+    const url = `${rootUrl}/content/list?page=1&size=${limit}&source_id=12&content_type=news&content_ids=&category_id=${cate}`;
+
+    const response = await got(url);
+    const data = response.data.data.list;
+    const list = data.map((item) => ({
+        title: item.title,
+        pubDate: parseDate(item.published),
+        author: item.author,
+        link: `https://www.diandong.com/news/${item.contentid}.html`,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const $ = cheerio.load(detailResponse.data);
+
+                item.description = $('#gallery-selector').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `电动邦 - ${titleMap[cate]}`,
+        link: 'https://www.diandong.com/news',
+        item: items,
+    };
+};

package/lib/v2/diandong/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'diandong.com': {
+        _name: '电动邦',
+        '.': [
+            {
+                title: '资讯',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/news'],
+                target: '/diandong/news/:cate',
+            },
+            {
+                title: '电动号',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/news/ddh'],
+                target: '/diandong/ddh/:cate',
+            },
+        ],
+    },
+};

package/lib/v2/diandong/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/news/:cate?', require('./news'));
+    router.get('/ddh/:cate?', require('./ddh'));
+};

package/lib/{routes → v2}/dongqiudi/daily.js

@@ -17,24 +17,21 @@ module.exports = async (ctx) => {
     const proList = [];
 
     const out = await Promise.all(
-        list.map(async (item) => {
+        list.map((item) => {
             const $ = cheerio.load(item);
             const title = $('a').text();
             const itemUrl = host + $('a').attr('href');
 
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
+            return ctx.cache.tryGet(itemUrl, () => {
+                const single = {
+                    title,
+                    link: itemUrl,
+                };
 
-            const single = {
-                title,
-                link: itemUrl,
-            };
-
-            const es = got.get(itemUrl);
-            proList.push(es);
-            return Promise.resolve(single);
+                const es = got.get(itemUrl);
+                proList.push(es);
+                return single;
+            });
         })
     );
 

package/lib/v2/dongqiudi/maintainer.js

@@ -0,0 +1,8 @@
+module.exports = {
+    '/daily': ['HenryQW'],
+    '/player_news/:id': ['HenryQW'],
+    '/result/:team': ['HenryQW'],
+    '/special/:id': ['dxmpalb'],
+    '/team_news/:team': ['HenryQW'],
+    '/top_news/:id?': ['HendricksZheng'],
+};

package/lib/v2/dongqiudi/radar.js

@@ -0,0 +1,45 @@
+module.exports = {
+    'dongqiudi.com': {
+        _name: '懂球帝',
+        m: [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/home/:id'],
+                target: '/dongqiudi/top_news/:id',
+            },
+        ],
+        www: [
+            {
+                title: '专题',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/special/:id'],
+                target: '/dongqiudi/special/:id',
+            },
+            {
+                title: '早报',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/special/48'],
+                target: '/dongqiudi/daily',
+            },
+            {
+                title: '足球赛果',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/team/*team'],
+                target: (params) => `/dongqiudi/result/${params.team.replace('.html', '')}`,
+            },
+            {
+                title: '球队新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/team/*team'],
+                target: (params) => `/dongqiudi/team_news/${params.team.replace('.html', '')}`,
+            },
+            {
+                title: '球员新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/player/*id'],
+                target: (params) => `/dongqiudi/player_news/${params.id.replace('.html', '')}`,
+            },
+        ],
+    },
+};

package/lib/{routes → v2}/dongqiudi/result.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const { JSDOM } = require('jsdom');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const team = ctx.params.team;
@@ -18,7 +19,7 @@ module.exports = async (ctx) => {
         title: `${result.match_title} ${result.team_A_name} ${result.fs_A}-${result.fs_B} ${result.team_B_name}`,
         guid: result.match_id,
         link: result.scheme.replace('dongqiudi:///game/', 'https://www.dongqiudi.com/liveDetail/'),
-        pubDate: result.start_time,
+        pubDate: parseDate(result.start_time),
     }));
 
     ctx.state.data = {

package/lib/v2/dongqiudi/router.js

@@ -0,0 +1,8 @@
+module.exports = (router) => {
+    router.get('/daily', require('./daily'));
+    router.get('/player_news/:id', require('./player_news'));
+    router.get('/result/:team', require('./result'));
+    router.get('/special/:id', require('./special'));
+    router.get('/team_news/:team', require('./team_news'));
+    router.get('/top_news/:id?', require('./top_news'));
+};

package/lib/{routes → v2}/dongqiudi/top_news.js

@@ -2,31 +2,28 @@ const got = require('@/utils/got');
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
-    const id = ctx.params.id || 1;
-    const response = await got.get(`https://api.dongqiudi.com/app/tabs/iphone/${id}.json?mark=gif&version=576`);
+    const id = ctx.params.id ?? 1;
+    const response = await got(`https://api.dongqiudi.com/app/tabs/iphone/${id}.json?mark=gif&version=576`);
     const data = response.data.articles;
     const label = response.data.label;
 
     const proList = [];
 
     const out = await Promise.all(
-        data.map(async (item) => {
+        data.map((item) => {
             const title = item.title;
             const itemUrl = `https://www.dongqiudi.com/news/${item.id}.html`;
 
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
+            return ctx.cache.tryGet(itemUrl, () => {
+                const single = {
+                    title,
+                    link: itemUrl,
+                };
 
-            const single = {
-                title,
-                link: itemUrl,
-            };
-
-            const es = got.get(itemUrl);
-            proList.push(es);
-            return Promise.resolve(single);
+                const es = got(itemUrl);
+                proList.push(es);
+                return single;
+            });
         })
     );
 

package/lib/{routes → v2}/dongqiudi/utils.js

@@ -1,11 +1,11 @@
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
-const url = require('url');
 const { JSDOM } = require('jsdom');
+const { parseDate } = require('@/utils/parse-date');
 
 const ProcessVideo = (content) => {
     content('div.video').each((i, v) => {
-        let link = url.parse(v.attribs.src);
+        let link = new URL(v.attribs.src);
         if (link.host === 'm.miguvideo.com') {
             content(`<a href="${link.href}"> ▶️ 观看视频 </a><br>`).insertAfter(v);
             content(v).remove();
@@ -13,8 +13,6 @@ const ProcessVideo = (content) => {
             link = v.attribs.src;
             switch (v.attribs.site) {
                 case 'qiniu':
-                    // 临时解决七牛 CDN 证书过期
-                    link = link.replace('https://', 'http://');
                     content(`<video width="100%" controls="controls"> <source src="${link}" type="video/mp4"> Your RSS reader does not support video playback. </video>`).insertAfter(v);
                     content(v).remove();
                     break;
@@ -31,7 +29,7 @@ const ProcessVideo = (content) => {
     // Process iframes
     content('iframe.media-iframe, .edui-faked-video').each((i, v) => {
         const link = v.attribs.src;
-        if (link.startsWith('http://ssports.iqiyi.com')) {
+        if (link.startsWith('http://ssports.iqiyi.com/')) {
             content(`<a href="${link.link}"> ▶️ 观看视频 </a><br>`).insertAfter(v);
         }
 
@@ -49,6 +47,20 @@ const ProcessHref = (content) => {
     });
 };
 
+const ProcessImg = (content) => {
+    content.each((_, img) => {
+        if (img.attribs['data-gif-src'] && img.attribs['data-gif-src'].length) {
+            img.attribs = { src: img.attribs['data-gif-src'] };
+        }
+        if (img.attribs['orig-src'] && img.attribs['orig-src'].length) {
+            img.attribs.src = img.attribs['orig-src'];
+            delete img.attribs['orig-src'];
+            delete img.attribs['data-src'];
+        }
+        img.attribs.src = img.attribs.src.includes('?watermark') ? img.attribs.src.split('?watermark')[0] : img.attribs.src;
+    });
+};
+
 const ProcessFeed = async (ctx, link, type) => {
     const got_ins = got.extend({
         headers: {
@@ -72,39 +84,31 @@ const ProcessFeed = async (ctx, link, type) => {
     } else if (type === 'player') {
         name = data.playerDetail.base_info.person_name;
     }
+    const proList = [];
 
     const out = await Promise.all(
-        list.map(async (item) => {
-            const res = await got_ins.get(item.share.replace('www', 'm').replace('articles', 'article'));
-
-            const $ = cheerio.load(res.data);
-
-            const author = $('body > header a').text();
-
-            const content = $('article');
-
-            ProcessVideo($);
-
-            $('.data-gif-src').each((j, y) => {
-                if (y.attribs['data-gif-src']) {
-                    y.attribs = { src: y.attribs['data-gif-src'] };
-                }
+        list.map((item) => {
+            const title = item.title;
+            const link = `https://www.dongqiudi.com/news/${item.id}.html`;
+
+            return ctx.cache.tryGet(link, () => {
+                const single = {
+                    title,
+                    link,
+                };
+
+                const es = got(link);
+                proList.push(es);
+                return single;
             });
-
-            ProcessHref(content.find('a'));
-
-            const single = {
-                title: item.title,
-                guid: item.id,
-                link: `https://www.dongqiudi.com/news/${item.id}.html`,
-                description: content.html(),
-                pubDate: new Date(item.show_time * 1000).toUTCString(),
-                author,
-            };
-            return Promise.resolve(single);
         })
     );
 
+    const responses = await got.all(proList);
+    for (let i = 0; i < proList.length; i++) {
+        ProcessFeedType2(out[i], responses[i].data);
+    }
+
     ctx.state.data = {
         title: `${name} - 相关新闻`,
         link,
@@ -119,12 +123,18 @@ const ProcessFeedType2 = (item, response) => {
 
     const data = dom.window.__NUXT__.data[0].newData;
 
+    // filter out undefined item
+    if (!data) {
+        return;
+    }
+
     if (Object.keys(data).length > 0) {
         const body = ProcessVideo(cheerio.load(data.body));
         ProcessHref(body('a'));
+        ProcessImg(body('img'));
         item.description = body.html();
         item.author = data.writer;
-        item.pubDate = new Date(data.show_time * 1000).toUTCString();
+        item.pubDate = parseDate(data.show_time * 1000);
     }
 };
 
@@ -133,4 +143,5 @@ module.exports = {
     ProcessFeed,
     ProcessFeedType2,
     ProcessHref,
+    ProcessImg,
 };

package/lib/v2/douyin/hashtag.js

@@ -0,0 +1,129 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const config = require('@/config').value;
+const { fallback, queryToBoolean } = require('@/utils/readable-social');
+
+const templates = {
+    desc: path.join(__dirname, 'templates/desc.art'),
+    cover: path.join(__dirname, 'templates/cover.art'),
+    embed: path.join(__dirname, 'templates/embed.art'),
+    iframe: path.join(__dirname, 'templates/iframe.art'),
+};
+
+const resolveUrl = (url, tls = true, forceResolve = false) => {
+    if (!url) {
+        return '';
+    }
+    if (url.startsWith('//')) {
+        return (tls ? 'https:' : 'http:') + url;
+    }
+    if (forceResolve && !url.match(/^https?:\/\//)) {
+        return (tls ? 'https://' : 'http://') + url;
+    }
+    return url;
+};
+
+const proxyVideo = (url, proxy) => {
+    if (!(url && proxy)) {
+        return url + '';
+    }
+    if (proxy.includes('?')) {
+        if (!proxy.endsWith('=')) {
+            proxy += '=';
+        }
+        return proxy + encodeURIComponent(url);
+    } else {
+        if (!proxy.endsWith('/')) {
+            proxy += '/';
+        }
+        return proxy + url;
+    }
+};
+
+const getOriginAvatar = (url) =>
+    resolveUrl(url)
+        .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
+        .replace(/~\w+_\d+x\d+/g, '');
+
+module.exports = async (ctx) => {
+    const { cid } = ctx.params;
+    if (isNaN(cid)) {
+        throw Error('Invalid tag ID. Tag ID should be a number.');
+    }
+    const routeParams = Object.fromEntries(new URLSearchParams(ctx.params.routeParams));
+    const embed = fallback(undefined, queryToBoolean(routeParams.embed), false); // embed video
+    const iframe = fallback(undefined, queryToBoolean(routeParams.iframe), false); // embed video in iframe
+    const relay = resolveUrl(routeParams.relay, true, true); // embed video behind a reverse proxy
+
+    const tagUrl = `https://www.douyin.com/hashtag/${cid}`;
+
+    const tagData = await ctx.cache.tryGet(
+        `douyin:hashtag:${cid}`,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            const page = await browser.newPage();
+            await page.goto(tagUrl, {
+                waitUntil: 'domcontentloaded',
+            });
+            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            await browser.close();
+
+            const $ = cheerio.load(html);
+            const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').text()));
+            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail && renderData[key].defaultData);
+            return renderData[dataKey];
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const tagInfo = tagData.topicDetail;
+    const tagName = tagInfo.chaName;
+    const userAvatar = getOriginAvatar(tagInfo.hashtagProfile);
+
+    const posts = tagData.defaultData;
+    const items = posts.map((post) => {
+        // parse video
+        let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
+        if (relay) {
+            videoList = videoList.map((item) => proxyVideo(item, relay));
+        }
+        let duration = post.video && post.video.duration;
+        duration = duration && duration / 1000;
+        let img;
+        // if (!embed) {
+        //     img = post.video && post.video.dynamicCover; // dynamic cover (webp)
+        // }
+        img =
+            img ||
+            (post.video &&
+                ((post.video.coverUrlList && post.video.coverUrlList[post.video.coverUrlList.length - 1]) || // HD
+                    post.video.originCover || // LD
+                    post.video.cover)); // even more LD
+        img = img && resolveUrl(img);
+
+        // render description
+        const desc = post.desc && post.desc.replace(/\n/g, '<br>');
+        let media = art(embed && videoList ? templates.embed : templates.cover, { img, videoList, duration });
+        media = embed && videoList && iframe ? art(templates.iframe, { content: media }) : media; // warp in iframe
+        const description = art(templates.desc, { desc, media });
+
+        return {
+            title: post.desc,
+            description,
+            link: `https://www.douyin.com/video/${post.awemeId}`,
+            pubDate: parseDate(post.createTime * 1000),
+            category: post.textExtra.map((extra) => extra.hashtagName),
+        };
+    });
+
+    ctx.state.data = {
+        title: tagName,
+        description: `${tagInfo.viewCount} 次播放`,
+        image: userAvatar,
+        link: tagUrl,
+        item: items,
+    };
+};

package/lib/v2/douyin/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/user/:uid/:routeParams?': ['Max-Tortoise', 'Rongronggg9'],
+};

package/lib/v2/douyin/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'douyin.com': {
+        _name: '抖音',
+        '.': [
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/social-media.html#dou-yin',
+                source: '/hashtag/:cid',
+                target: '/douyin/hashtag/:cid',
+            },
+            {
+                title: '博主',
+                docs: 'https://docs.rsshub.app/social-media.html#dou-yin',
+                source: '/user/:uid',
+                target: '/douyin/user/:uid',
+            },
+        ],
+    },
+};

package/lib/v2/douyin/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/hashtag/:cid/:routeParams?', require('./hashtag'));
+    router.get('/user/:uid/:routeParams?', require('./user'));
+};

package/lib/v2/douyin/templates/cover.art

@@ -0,0 +1,15 @@
+{{ if img }}
+  {{ if videoList }}
+  <a href="{{videoList[0]}}" rel="noreferrer">
+  {{ /if }}
+    <img src="{{img}}" style="width: 50%;">
+  {{ if videoList }}
+  </a>
+  {{ /if }}
+{{ /if }}
+{{ if img && videoList }}
+  <br><br>
+{{ /if }}
+{{ if videoList }}
+  <a href="{{videoList[0]}}" rel="noreferrer">视频直链</a>
+{{ /if }}

package/lib/v2/douyin/templates/desc.art

@@ -0,0 +1,3 @@
+{{@ desc }}
+<br><br>
+{{@ media }}

package/lib/v2/douyin/templates/embed.art

@@ -0,0 +1,13 @@
+<video controls preload="none" referrerpolicy="no-referrer"
+style="width:50%"
+{{ if img }}
+poster="{{ img }}"
+{{ /if }}
+{{ if duration }}
+duration="{{ duration }}"
+{{ /if }}
+>
+  {{ each videoList source }}
+  <source referrerpolicy="no-referrer" src="{{ source }}">
+  {{ /each }}
+</video>

package/lib/v2/douyin/templates/iframe.art

@@ -0,0 +1,14 @@
+<iframe referrerpolicy="no-referrer" width=100% height=150vh frameborder=0 marginheight=0 marginwidth=0
+style="border:0; margin:0; padding:0; width:100%; height:150vh;"
+srcdoc="
+&lt;!DOCTYPE html&gt;
+&lt;html&gt;
+  &lt;head&gt;
+    &lt;meta name=&quot;referrer&quot; content=&quot;no-referrer&quot;&gt;
+  &lt;/head&gt;
+  &lt;body&gt;
+    {{ content }}
+  &lt;/body&gt;
+&lt;/html&gt;
+">
+</iframe>