rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/v2/domp4/utils.js

@@ -0,0 +1,72 @@
+/**
+ * trackers from https://www.domp4.cc/Style/2020/js/base.js?v=2
+ */
+const magnetTrackers = [
+    'https://tracker.iriseden.fr:443/announce',
+    'https://tr.highstar.shop:443/announce',
+    'https://tr.fuckbitcoin.xyz:443/announce',
+    'https://tr.doogh.club:443/announce',
+    'https://tr.burnabyhighstar.com:443/announce',
+    'https://t.btcland.xyz:443/announce',
+    'http://vps02.net.orel.ru:80/announce',
+    'https://tracker.kuroy.me:443/announce',
+    'http://tr.cili001.com:8070/announce',
+    'http://t.overflow.biz:6969/announce',
+    'http://t.nyaatracker.com:80/announce',
+    'http://open.acgnxtracker.com:80/announce',
+    'http://nyaa.tracker.wf:7777/announce',
+    'http://home.yxgz.vip:6969/announce',
+    'http://buny.uk:6969/announce',
+    'https://tracker.tamersunion.org:443/announce',
+    'https://tracker.nanoha.org:443/announce',
+    'https://tracker.loligirl.cn:443/announce',
+    'udp://bubu.mapfactor.com:6969/announce',
+    'http://share.camoe.cn:8080/announce',
+    'udp://movies.zsw.ca:6969/announce',
+    'udp://ipv4.tracker.harry.lu:80/announce',
+    'udp://tracker.sylphix.com:6969/announce',
+    'http://95.216.22.207:9001/announce',
+];
+
+/**
+ * compose magnet url with trackers
+ */
+function composeMagnetUrl(magnet, trackers = magnetTrackers) {
+    return `${magnet}&tr=${trackers.join('&tr=')}`;
+}
+
+/**
+ * method for generating magnet url
+ * from the detail page source
+ */
+function decodeCipherText(p, a, c, k, e, d) {
+    e = function (c) {
+        return (c < a ? '' : e(parseInt(c / a))) + ((c = c % a) > 35 ? String.fromCharCode(c + 29) : c.toString(36));
+    };
+    if (!''.replace(/^/, String)) {
+        while (c--) {
+            d[e(c)] = k[c] || e(c);
+        }
+        k = [
+            function (e) {
+                return d[e];
+            },
+        ];
+        e = function () {
+            return '\\w+';
+        };
+        c = 1;
+    }
+    while (c--) {
+        if (k[c]) {
+            p = p.replace(new RegExp('\\b' + e(c) + '\\b', 'g'), k[c]);
+        }
+    }
+    return p;
+}
+
+module.exports = {
+    magnetTrackers,
+    composeMagnetUrl,
+    decodeCipherText,
+};

package/lib/v2/dongqiudi/daily.js

@@ -0,0 +1,47 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const utils = require('./utils');
+
+module.exports = async (ctx) => {
+    const response = await got.get('https://www.dongqiudi.com/special/48');
+
+    const $ = cheerio.load(response.data);
+
+    const host = 'https://www.dongqiudi.com';
+
+    const list = $('.detail.special ul li h3')
+        .slice(0, 5)
+        .get()
+        .filter((e) => cheerio.load(e).text().length > 3);
+
+    const proList = [];
+
+    const out = await Promise.all(
+        list.map((item) => {
+            const $ = cheerio.load(item);
+            const title = $('a').text();
+            const itemUrl = host + $('a').attr('href');
+
+            return ctx.cache.tryGet(itemUrl, () => {
+                const single = {
+                    title,
+                    link: itemUrl,
+                };
+
+                const es = got.get(itemUrl);
+                proList.push(es);
+                return single;
+            });
+        })
+    );
+
+    const responses = await got.all(proList);
+    for (let i = 0; i < proList.length; i++) {
+        utils.ProcessFeedType2(out[i], responses[i].data);
+    }
+    ctx.state.data = {
+        title: '懂球帝早报',
+        link: 'http://www.dongqiudi.com/special/48',
+        item: out.filter((e) => e.description !== undefined),
+    };
+};

package/lib/v2/dongqiudi/maintainer.js

@@ -0,0 +1,8 @@
+module.exports = {
+    '/daily': ['HenryQW'],
+    '/player_news/:id': ['HenryQW'],
+    '/result/:team': ['HenryQW'],
+    '/special/:id': ['dxmpalb'],
+    '/team_news/:team': ['HenryQW'],
+    '/top_news/:id?': ['HendricksZheng'],
+};

package/lib/v2/dongqiudi/radar.js

@@ -0,0 +1,45 @@
+module.exports = {
+    'dongqiudi.com': {
+        _name: '懂球帝',
+        m: [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/home/:id'],
+                target: '/dongqiudi/top_news/:id',
+            },
+        ],
+        www: [
+            {
+                title: '专题',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/special/:id'],
+                target: '/dongqiudi/special/:id',
+            },
+            {
+                title: '早报',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/special/48'],
+                target: '/dongqiudi/daily',
+            },
+            {
+                title: '足球赛果',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/team/*team'],
+                target: (params) => `/dongqiudi/result/${params.team.replace('.html', '')}`,
+            },
+            {
+                title: '球队新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/team/*team'],
+                target: (params) => `/dongqiudi/team_news/${params.team.replace('.html', '')}`,
+            },
+            {
+                title: '球员新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#dong-qiu-di',
+                source: ['/player/*id'],
+                target: (params) => `/dongqiudi/player_news/${params.id.replace('.html', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/dongqiudi/result.js

@@ -0,0 +1,30 @@
+const got = require('@/utils/got');
+const { JSDOM } = require('jsdom');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const team = ctx.params.team;
+    const link = `https://www.dongqiudi.com/team/${team}.html`;
+
+    const response = await got.get(link);
+    const dom = new JSDOM(response.data, {
+        runScripts: 'dangerously',
+    });
+    const data = dom.window.__NUXT__.data[0];
+    const resultData = data.teamScheduleData.filter((data) => data.fs_A && data.fs_B);
+
+    const teamName = data.teamDetail.base_info.team_name;
+
+    const out = resultData.map((result) => ({
+        title: `${result.match_title} ${result.team_A_name} ${result.fs_A}-${result.fs_B} ${result.team_B_name}`,
+        guid: result.match_id,
+        link: result.scheme.replace('dongqiudi:///game/', 'https://www.dongqiudi.com/liveDetail/'),
+        pubDate: parseDate(result.start_time),
+    }));
+
+    ctx.state.data = {
+        title: `${teamName} 比赛结果`,
+        link,
+        item: out.slice(out.length - 10, out.length),
+    };
+};

package/lib/v2/dongqiudi/router.js

@@ -0,0 +1,8 @@
+module.exports = (router) => {
+    router.get('/daily', require('./daily'));
+    router.get('/player_news/:id', require('./player_news'));
+    router.get('/result/:team', require('./result'));
+    router.get('/special/:id', require('./special'));
+    router.get('/team_news/:team', require('./team_news'));
+    router.get('/top_news/:id?', require('./top_news'));
+};

package/lib/v2/dongqiudi/top_news.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const utils = require('./utils');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? 1;
+    const response = await got(`https://api.dongqiudi.com/app/tabs/iphone/${id}.json?mark=gif&version=576`);
+    const data = response.data.articles;
+    const label = response.data.label;
+
+    const proList = [];
+
+    const out = await Promise.all(
+        data.map((item) => {
+            const title = item.title;
+            const itemUrl = `https://www.dongqiudi.com/news/${item.id}.html`;
+
+            return ctx.cache.tryGet(itemUrl, () => {
+                const single = {
+                    title,
+                    link: itemUrl,
+                };
+
+                const es = got(itemUrl);
+                proList.push(es);
+                return single;
+            });
+        })
+    );
+
+    const responses = await got.all(proList);
+    for (let i = 0; i < proList.length; i++) {
+        utils.ProcessFeedType2(out[i], responses[i].data);
+    }
+    ctx.state.data = {
+        title: `懂球帝 - ${label}`,
+        link: 'http://dongqiudi.com/news',
+        item: out.filter((e) => e !== undefined),
+    };
+};

package/lib/v2/dongqiudi/utils.js

@@ -0,0 +1,147 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { JSDOM } = require('jsdom');
+const { parseDate } = require('@/utils/parse-date');
+
+const ProcessVideo = (content) => {
+    content('div.video').each((i, v) => {
+        let link = new URL(v.attribs.src);
+        if (link.host === 'm.miguvideo.com') {
+            content(`<a href="${link.href}"> ▶️ 观看视频 </a><br>`).insertAfter(v);
+            content(v).remove();
+        } else {
+            link = v.attribs.src;
+            switch (v.attribs.site) {
+                case 'qiniu':
+                    content(`<video width="100%" controls="controls"> <source src="${link}" type="video/mp4"> Your RSS reader does not support video playback. </video>`).insertAfter(v);
+                    content(v).remove();
+                    break;
+                case 'youku':
+                    content(`<iframe height='100%' width='100%' src='${link}' frameborder=0 scrolling=no webkitallowfullscreen=true allowfullscreen=true></iframe>`).insertAfter(v);
+                    content(v).remove();
+                    break;
+                default:
+                    break;
+            }
+        }
+    });
+
+    // Process iframes
+    content('iframe.media-iframe, .edui-faked-video').each((i, v) => {
+        const link = v.attribs.src;
+        if (link.startsWith('http://ssports.iqiyi.com/')) {
+            content(`<a href="${link.link}"> ▶️ 观看视频 </a><br>`).insertAfter(v);
+        }
+
+        content(v).remove();
+    });
+
+    return content;
+};
+
+const ProcessHref = (content) => {
+    content.each((j, y) => {
+        if (y.attribs.href) {
+            y.attribs.href = y.attribs.href.replace('dongqiudi:///news', 'https://www.dongqiudi.com/share/article');
+        }
+    });
+};
+
+const ProcessImg = (content) => {
+    content.each((_, img) => {
+        if (img.attribs['data-gif-src'] && img.attribs['data-gif-src'].length) {
+            img.attribs = { src: img.attribs['data-gif-src'] };
+        }
+        if (img.attribs['orig-src'] && img.attribs['orig-src'].length) {
+            img.attribs.src = img.attribs['orig-src'];
+            delete img.attribs['orig-src'];
+            delete img.attribs['data-src'];
+        }
+        img.attribs.src = img.attribs.src.includes('?watermark') ? img.attribs.src.split('?watermark')[0] : img.attribs.src;
+    });
+};
+
+const ProcessFeed = async (ctx, link, type) => {
+    const got_ins = got.extend({
+        headers: {
+            Referer: link,
+        },
+    });
+
+    const response = await got_ins.get(link);
+
+    let name;
+
+    const dom = new JSDOM(response.data, {
+        runScripts: 'dangerously',
+    });
+
+    const data = dom.window.__NUXT__.data[0];
+    const list = data.relatedNewsList;
+
+    if (type === 'team') {
+        name = data.teamDetail.base_info.team_name;
+    } else if (type === 'player') {
+        name = data.playerDetail.base_info.person_name;
+    }
+    const proList = [];
+
+    const out = await Promise.all(
+        list.map((item) => {
+            const title = item.title;
+            const link = `https://www.dongqiudi.com/news/${item.id}.html`;
+
+            return ctx.cache.tryGet(link, () => {
+                const single = {
+                    title,
+                    link,
+                };
+
+                const es = got(link);
+                proList.push(es);
+                return single;
+            });
+        })
+    );
+
+    const responses = await got.all(proList);
+    for (let i = 0; i < proList.length; i++) {
+        ProcessFeedType2(out[i], responses[i].data);
+    }
+
+    ctx.state.data = {
+        title: `${name} - 相关新闻`,
+        link,
+        item: out,
+    };
+};
+
+const ProcessFeedType2 = (item, response) => {
+    const dom = new JSDOM(response, {
+        runScripts: 'dangerously',
+    });
+
+    const data = dom.window.__NUXT__.data[0].newData;
+
+    // filter out undefined item
+    if (!data) {
+        return;
+    }
+
+    if (Object.keys(data).length > 0) {
+        const body = ProcessVideo(cheerio.load(data.body));
+        ProcessHref(body('a'));
+        ProcessImg(body('img'));
+        item.description = body.html();
+        item.author = data.writer;
+        item.pubDate = parseDate(data.show_time * 1000);
+    }
+};
+
+module.exports = {
+    ProcessVideo,
+    ProcessFeed,
+    ProcessFeedType2,
+    ProcessHref,
+    ProcessImg,
+};

package/lib/v2/douban/people/status.js

@@ -1,6 +1,7 @@
 const querystring = require('querystring');
 const got = require('@/utils/got');
 const { fallback, queryToBoolean, queryToInteger } = require('@/utils/readable-social');
+const config = require('@/config').value;
 
 const headers = { Referer: `https://m.douban.com/` };
 
@@ -466,8 +467,15 @@ async function getFullTextItems(ctx, items) {
 module.exports = async (ctx) => {
     const userid = ctx.params.userid;
     const url = `https://m.douban.com/rexxar/api/v2/status/user_timeline/${userid}`;
-    const response = await got({ url, headers });
-    const items = response.data.items;
+    const items = await ctx.cache.tryGet(
+        url,
+        async () => {
+            const _r = await got({ url, headers });
+            return _r.data.items;
+        },
+        config.cache.routeExpire,
+        false
+    );
 
     if (items) {
         await getFullTextItems(ctx, items);

package/lib/v2/douban/people/wish.js

@@ -13,6 +13,8 @@ module.exports = async (ctx) => {
     const torrentMinRating = routeParams.torrentMinRating ? parseFloat(routeParams.torrentMinRating) : 0.5;
     const torrentSkipNoMatch = routeParams.torrentSkipNoMatch === 'true';
 
+    let userName;
+
     const pageSize = 15;
     const pagesCount = routeParams.pagesCount ? parseInt(routeParams.pagesCount) : 1;
     const tasks = [];
@@ -20,51 +22,61 @@ module.exports = async (ctx) => {
         const url = `https://movie.douban.com/people/${userid}/wish?start=${page * pageSize}`;
 
         tasks.push(
-            got({
-                method: 'GET',
-                url,
-                headers: {
-                    Referer: url,
-                    Cookie: config.douban.cookie || '',
-                },
-            }).then((response) => {
-                const data = response.data;
-                const $ = cheerio.load(data);
-                const list = $('div.article > div.grid-view > div.item');
+            ctx.cache
+                .tryGet(
+                    url,
+                    async () => {
+                        const _r = await got({
+                            method: 'GET',
+                            url,
+                            headers: {
+                                Referer: url,
+                                Cookie: config.douban.cookie || '',
+                            },
+                        });
+                        return _r.data;
+                    },
+                    config.cache.routeExpire,
+                    false
+                )
+                .then((data) => {
+                    const $ = cheerio.load(data);
+                    const list = $('div.article > div.grid-view > div.item');
+                    userName = userName || $('div.side-info-txt > h3').text();
 
-                if (list) {
-                    return Promise.all(
-                        list.get().map(async (item) => {
-                            item = $(item);
-                            const itemPicUrl = item.find('.pic a img').attr('src');
-                            const info = item.find('.info');
-                            const title = info.find('ul li.title a').text();
-                            const url = info.find('ul li.title a').attr('href');
-                            const titles = title.split('/').filter((title) => title.trim());
-                            const day = info.find('ul li .date').text().trim();
-                            const rssItem = {
-                                title: titles[0],
-                                description: `${info.find('.intro').text()}<br><img src="${itemPicUrl}">`,
-                                link: url,
-                                pubDate: new Date(day),
-                            };
+                    if (list) {
+                        return Promise.all(
+                            list.get().map(async (item) => {
+                                item = $(item);
+                                const itemPicUrl = item.find('.pic a img').attr('src');
+                                const info = item.find('.info');
+                                const title = info.find('ul li.title a').text();
+                                const url = info.find('ul li.title a').attr('href');
+                                const titles = title.split('/').filter((title) => title.trim());
+                                const day = info.find('ul li .date').text().trim();
+                                const rssItem = {
+                                    title: titles[0],
+                                    description: `${info.find('.intro').text()}<br><img src="${itemPicUrl}">`,
+                                    link: url,
+                                    pubDate: new Date(day),
+                                };
 
-                            if (routeParams.torrentProvider) {
-                                const category = url.indexOf('movie.douban.com') > 0 ? 'Movies' : 'TV';
-                                const keyword = titles.length > 1 ? titles[1] : titles[0];
-                                let keywords = [keyword];
-                                if (routeParams.torrentQuery) {
-                                    keywords = keywords.concat(routeParams.torrentQuery.split(','));
+                                if (routeParams.torrentProvider) {
+                                    const category = url.indexOf('movie.douban.com') > 0 ? 'Movies' : 'TV';
+                                    const keyword = titles.length > 1 ? titles[1] : titles[0];
+                                    let keywords = [keyword];
+                                    if (routeParams.torrentQuery) {
+                                        keywords = keywords.concat(routeParams.torrentQuery.split(','));
+                                    }
+                                    const result = await torrent.get(providers, routeParams, keywords, category, torrentMinSeeds, torrentMinRating);
+                                    return Object.assign(rssItem, result);
+                                } else {
+                                    return rssItem;
                                 }
-                                const result = await torrent.get(providers, routeParams, keywords, category, torrentMinSeeds, torrentMinRating);
-                                return Object.assign(rssItem, result);
-                            } else {
-                                return rssItem;
-                            }
-                        })
-                    );
-                }
-            })
+                            })
+                        );
+                    }
+                })
         );
     }
 
@@ -73,7 +85,7 @@ module.exports = async (ctx) => {
         items = items.filter((item) => item.enclosure_url && item.enclosure_url.length > 0);
     }
     ctx.state.data = {
-        title: `豆瓣想看 - ${userid}`,
+        title: `豆瓣想看 - ${userName || userid}`,
         link: `https://movie.douban.com/people/${userid}/wish`,
         item: items,
     };

package/lib/v2/douyin/hashtag.js

@@ -0,0 +1,91 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const config = require('@/config').value;
+const { fallback, queryToBoolean } = require('@/utils/readable-social');
+const { templates, resolveUrl, proxyVideo, getOriginAvatar } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { cid } = ctx.params;
+    if (isNaN(cid)) {
+        throw Error('Invalid tag ID. Tag ID should be a number.');
+    }
+    const routeParams = Object.fromEntries(new URLSearchParams(ctx.params.routeParams));
+    const embed = fallback(undefined, queryToBoolean(routeParams.embed), false); // embed video
+    const iframe = fallback(undefined, queryToBoolean(routeParams.iframe), false); // embed video in iframe
+    const relay = resolveUrl(routeParams.relay, true, true); // embed video behind a reverse proxy
+
+    const tagUrl = `https://www.douyin.com/hashtag/${cid}`;
+
+    const tagData = await ctx.cache.tryGet(
+        `douyin:hashtag:${cid}`,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' ? request.continue() : request.abort();
+            });
+            await page.goto(tagUrl, {
+                waitUntil: 'domcontentloaded',
+            });
+            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            await browser.close();
+
+            const $ = cheerio.load(html);
+            const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').text()));
+            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail && renderData[key].defaultData);
+            return renderData[dataKey];
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const tagInfo = tagData.topicDetail;
+    const tagName = tagInfo.chaName;
+    const userAvatar = getOriginAvatar(tagInfo.hashtagProfile);
+
+    const posts = tagData.defaultData;
+    const items = posts.map((post) => {
+        // parse video
+        let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
+        if (relay) {
+            videoList = videoList.map((item) => proxyVideo(item, relay));
+        }
+        let duration = post.video && post.video.duration;
+        duration = duration && duration / 1000;
+        let img;
+        // if (!embed) {
+        //     img = post.video && post.video.dynamicCover; // dynamic cover (webp)
+        // }
+        img =
+            img ||
+            (post.video &&
+                ((post.video.coverUrlList && post.video.coverUrlList[post.video.coverUrlList.length - 1]) || // HD
+                    post.video.originCover || // LD
+                    post.video.cover)); // even more LD
+        img = img && resolveUrl(img);
+
+        // render description
+        const desc = post.desc && post.desc.replace(/\n/g, '<br>');
+        let media = art(embed && videoList ? templates.embed : templates.cover, { img, videoList, duration });
+        media = embed && videoList && iframe ? art(templates.iframe, { content: media }) : media; // warp in iframe
+        const description = art(templates.desc, { desc, media });
+
+        return {
+            title: post.desc,
+            description,
+            link: `https://www.douyin.com/video/${post.awemeId}`,
+            pubDate: parseDate(post.createTime * 1000),
+            category: post.textExtra.map((extra) => extra.hashtagName),
+        };
+    });
+
+    ctx.state.data = {
+        title: tagName,
+        description: `${tagInfo.viewCount} 次播放`,
+        image: userAvatar,
+        link: tagUrl,
+        item: items,
+    };
+};