rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/v2/dmzj/radar.js

@@ -0,0 +1,67 @@
+module.exports = {
+    'dmzj.com': {
+        _name: '动漫之家',
+        news: [
+            {
+                title: '宅新闻',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/',
+                target: '/dmzj/news',
+            },
+            {
+                title: '漫画情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manhuaqingbao',
+                target: '/dmzj/news/manhuaqingbao',
+            },
+            {
+                title: '轻小说情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/qingxiaoshuoqingbao',
+                target: '/dmzj/news/qingxiaoshuoqingbao',
+            },
+            {
+                title: '动漫周边',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manhuazhoubian',
+                target: '/dmzj/news/manhuazhoubian',
+            },
+            {
+                title: '声优情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/shengyouqingbao',
+                target: '/dmzj/news/shengyouqingbao',
+            },
+            {
+                title: '音乐资讯',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/yinyuezixun',
+                target: '/dmzj/news/yinyuezixun',
+            },
+            {
+                title: '游戏资讯',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/youxizixun',
+                target: '/dmzj/news/youxizixun',
+            },
+            {
+                title: '美图欣赏',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/meituxinshang',
+                target: '/dmzj/news/meituxinshang',
+            },
+            {
+                title: '漫展情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manzhanqingbao',
+                target: '/dmzj/news/manzhanqingbao',
+            },
+            {
+                title: '大杂烩',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/dazahui',
+                target: '/dmzj/news/dazahui',
+            },
+        ],
+    },
+};

package/lib/v2/dmzj/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/news/:category?', require('./news'));
+};

package/lib/v2/docschina/jsweekly.js

@@ -0,0 +1,114 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const md = require('markdown-it')();
+
+module.exports = async (ctx) => {
+    const baseURL = 'https://docschina.org';
+    const { data: res } = await got(`${baseURL}/weekly/js/docs/`);
+
+    const $ = cheerio.load(res);
+
+    const title = $('head title').text();
+    const vendorLink = $('body script')
+        .toArray()
+        .find((i) =>
+            $(i)
+                .attr('src')
+                .match(/vendor/)
+        ).attribs.src;
+
+    const { data: vendor } = await got(`https:${vendorLink}`);
+    const env = vendor.match(/default.init\({env:"(.*?)"}\)/)[1]; // docschina-live-10765e
+    const dataVersion = vendor.match(/,dataVersion:"(\d{4}-\d{2}-\d{2})",/)[1]; // 2020-01-10
+
+    // const seqId = Math.random().toString(16).slice(2);
+    const { data: document } = await got.post('https://tcb-api.tencentcloudapi.com/web', {
+        headers: {
+            origin: 'https://docschina.org',
+            // referer: 'https://docschina.org/',
+            // 'x-sdk-version': '@cloudbase/js-sdk/1.3.3',
+            // 'x-seqid': seqId,
+        },
+        searchParams: {
+            env,
+        },
+        json: {
+            action: 'database.queryDocument',
+            collectionName: 'js_weekly_document',
+            dataVersion,
+            env,
+            limit: 100,
+            query: JSON.stringify({
+                path: '/',
+            }),
+            queryType: 'WHERE',
+            // seqId,
+        },
+    });
+
+    const { data: sidebar } = await got.post('https://tcb-api.tencentcloudapi.com/web', {
+        headers: {
+            origin: 'https://docschina.org',
+        },
+        searchParams: {
+            env,
+        },
+        json: {
+            action: 'database.queryDocument',
+            collectionName: 'js_weekly_sidebar',
+            dataVersion,
+            env,
+            limit: 100,
+            query: JSON.stringify({
+                pages: {
+                    $in: [document.data.list[0]._id],
+                },
+            }),
+        },
+    });
+
+    const list = sidebar.data.list[0].value
+        .filter((i) => i.key !== 'home')
+        .map((i) => ({
+            title: i.title,
+            link: `${baseURL}/weekly/js${i.path}`,
+            key: i.key,
+        }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: document } = await got.post('https://tcb-api.tencentcloudapi.com/web', {
+                    headers: {
+                        origin: 'https://docschina.org',
+                    },
+                    searchParams: {
+                        env,
+                    },
+                    json: {
+                        action: 'database.queryDocument',
+                        collectionName: 'js_weekly_document',
+                        dataVersion,
+                        env,
+                        limit: 100,
+                        query: JSON.stringify({
+                            path: item.key,
+                        }),
+                    },
+                });
+
+                item.description = md.render(document.data.list[0].content);
+
+                delete item.key;
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title,
+        link: `${baseURL}/weekly/js/docs/`,
+        icon: 'https://docschina.org/favicon.ico',
+        item: items,
+    };
+};

package/lib/v2/docschina/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/jsweekly': ['daijinru'],
+};

package/lib/v2/docschina/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    docschina: {
+        _name: '印记中文',
+        '.': [
+            {
+                title: '周刊 - JavaScript',
+                docs: 'https://docs.rsshub.app/programming.html#yin-ji-zhong-wen-zhou-kan',
+                source: ['/weekly/js/*', '/weekly/js', '/'],
+                target: '/docschina/jsweekly',
+            },
+        ],
+    },
+};

package/lib/v2/docschina/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/jsweekly', require('./jsweekly'));
+};

package/lib/v2/domp4/detail.js

@@ -1,6 +1,6 @@
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
-const { decodeCipherText, composeMagnetUrl } = require('./utils');
+const { decodeCipherText, composeMagnetUrl, getUrlType } = require('./utils');
 
 const baseUrl = 'https://www.domp4.cc';
 
@@ -16,13 +16,17 @@ function getItemList($, detailUrl) {
     );
     const { downurls } = data.Data[0];
     return downurls.map((item) => {
-        const [title, magnet] = item.split('$');
+        const [title, downurl] = item.split('$');
+        const urlType = getUrlType(downurl);
+        // only magnet need compose trackers
+        const enclosureUrl = urlType === 'magnet' ? composeMagnetUrl(downurl) : downurl;
         return {
-            enclosure_url: composeMagnetUrl(magnet),
+            enclosure_url: enclosureUrl,
             enclosure_length: '',
             enclosure_type: 'application/x-bittorrent',
             title,
             link: detailUrl,
+            guid: `${title}-${urlType}`,
         };
     });
 }

package/lib/v2/domp4/utils.js

@@ -35,6 +35,19 @@ function composeMagnetUrl(magnet, trackers = magnetTrackers) {
     return `${magnet}&tr=${trackers.join('&tr=')}`;
 }
 
+/**
+ * method for download url type
+ */
+function getUrlType(url) {
+    if (url.startsWith('magnet:')) {
+        return 'magnet';
+    }
+    if (url.startsWith('ed2k:')) {
+        return 'ed2k';
+    }
+    return '';
+}
+
 /**
  * method for generating magnet url
  * from the detail page source
@@ -67,6 +80,7 @@ function decodeCipherText(p, a, c, k, e, d) {
 
 module.exports = {
     magnetTrackers,
+    getUrlType,
     composeMagnetUrl,
     decodeCipherText,
 };

package/lib/v2/dongqiudi/player_news.js

@@ -1,8 +1,7 @@
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
-    const id = ctx.params.id;
-    const link = `https://www.dongqiudi.com/player/${id}.html`;
+    const playerId = ctx.params.id;
 
-    await utils.ProcessFeed(ctx, link, 'player');
+    await utils.ProcessFeed(ctx, 'player', playerId);
 };

package/lib/v2/dongqiudi/result.js

@@ -6,7 +6,7 @@ module.exports = async (ctx) => {
     const team = ctx.params.team;
     const link = `https://www.dongqiudi.com/team/${team}.html`;
 
-    const response = await got.get(link);
+    const response = await got(link);
     const dom = new JSDOM(response.data, {
         runScripts: 'dangerously',
     });

package/lib/v2/dongqiudi/router.js

@@ -1,5 +1,5 @@
 module.exports = (router) => {
-    router.get('/daily', require('./daily'));
+    router.redirect('/daily', '/dongqiudi/special/48');
     router.get('/player_news/:id', require('./player_news'));
     router.get('/result/:team', require('./result'));
     router.get('/special/:id', require('./special'));

package/lib/v2/dongqiudi/special.js

@@ -4,44 +4,38 @@ const utils = require('./utils');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
-    const response = await got.get(`https://www.dongqiudi.com/special/${id}`);
+    const response = await got(`https://www.dongqiudi.com/special/${id}`);
 
     const $ = cheerio.load(response.data);
 
     const host = 'https://www.dongqiudi.com';
 
-    const list = $('.detail.special ul li h3').slice(0, 5).get();
-
-    const proList = [];
+    const list = $('.detail.special ul li h3')
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 5)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').text(),
+                link: host + item.find('a').attr('href'),
+            };
+        });
 
     const out = await Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const title = $('a').text();
-            const itemUrl = host + $('a').attr('href');
-
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const single = {
-                title,
-                link: itemUrl,
-            };
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+
+                utils.ProcessFeedType2(item, response);
 
-            const es = got.get(itemUrl);
-            proList.push(es);
-            return Promise.resolve(single);
-        })
+                return item;
+            })
+        )
     );
 
-    const responses = await got.all(proList);
-    for (let i = 0; i < proList.length; i++) {
-        utils.ProcessFeedType2(out[i], responses[i].data);
-    }
     ctx.state.data = {
-        title: `懂球帝专题-${id}`,
+        title: `懂球帝专题-${$('.detail.special h1').text()}`,
+        description: $('.detail.special h4').text(),
         link: `https://www.dongqiudi.com/special/${id}`,
         item: out.filter((e) => e !== undefined),
     };

package/lib/v2/dongqiudi/team_news.js

@@ -1,8 +1,7 @@
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
-    const team = ctx.params.team;
-    const link = `https://www.dongqiudi.com/team/${team}.html`;
+    const teamId = ctx.params.team;
 
-    await utils.ProcessFeed(ctx, link, 'team');
+    await utils.ProcessFeed(ctx, 'team', teamId);
 };

package/lib/v2/dongqiudi/top_news.js

@@ -1,39 +1,33 @@
 const got = require('@/utils/got');
 const utils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id ?? 1;
-    const response = await got(`https://api.dongqiudi.com/app/tabs/iphone/${id}.json?mark=gif&version=576`);
-    const data = response.data.articles;
-    const label = response.data.label;
+    const { data } = await got(`https://api.dongqiudi.com/app/tabs/web/${id}.json`);
+    const articles = data.articles;
+    const label = data.label;
 
-    const proList = [];
+    const list = articles.map((item) => ({
+        title: item.title,
+        link: `https://www.dongqiudi.com/articles/${item.id}.html`,
+        category: [item.category, ...(item.secondary_category ? item.secondary_category : [])],
+        pubDate: parseDate(item.show_time),
+    }));
 
     const out = await Promise.all(
-        data.map((item) => {
-            const title = item.title;
-            const itemUrl = `https://www.dongqiudi.com/news/${item.id}.html`;
-
-            return ctx.cache.tryGet(itemUrl, () => {
-                const single = {
-                    title,
-                    link: itemUrl,
-                };
-
-                const es = got(itemUrl);
-                proList.push(es);
-                return single;
-            });
-        })
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                utils.ProcessFeedType2(item, response);
+                return item;
+            })
+        )
     );
 
-    const responses = await got.all(proList);
-    for (let i = 0; i < proList.length; i++) {
-        utils.ProcessFeedType2(out[i], responses[i].data);
-    }
     ctx.state.data = {
         title: `懂球帝 - ${label}`,
-        link: 'http://dongqiudi.com/news',
+        link: `https://www.dongqiudi.com/articlesList/${id}`,
         item: out.filter((e) => e !== undefined),
     };
 };

package/lib/v2/dongqiudi/utils.js

@@ -61,57 +61,57 @@ const ProcessImg = (content) => {
     });
 };
 
-const ProcessFeed = async (ctx, link, type) => {
-    const got_ins = got.extend({
-        headers: {
-            Referer: link,
-        },
-    });
-
-    const response = await got_ins.get(link);
+const ProcessFeed = async (ctx, type, id) => {
+    const link = `https://www.dongqiudi.com/${type}/${id}.html`;
+    const apiUrl = `https://api.dongqiudi.com/v3/archive/app/channel/feeds`;
+    const { data: response } = await got(link);
 
     let name;
 
-    const dom = new JSDOM(response.data, {
+    const { window } = new JSDOM(response, {
         runScripts: 'dangerously',
     });
 
-    const data = dom.window.__NUXT__.data[0];
-    const list = data.relatedNewsList;
-
+    const typeInfo = window.__NUXT__.data[0][`${type}Detail`].base_info;
     if (type === 'team') {
-        name = data.teamDetail.base_info.team_name;
+        name = typeInfo.team_name;
     } else if (type === 'player') {
-        name = data.playerDetail.base_info.person_name;
+        name = typeInfo.person_name;
     }
-    const proList = [];
+
+    const { data } = await got(apiUrl, {
+        searchParams: {
+            id,
+            type,
+            size: 20,
+            platform: 'web',
+            version: '',
+        },
+    });
+
+    const list = data.data.articles.map((article) => ({
+        title: article.title,
+        link: `https://www.dongqiudi.com/articles/${article.id}.html`,
+        category: [article.category, ...(article.secondary_category ? article.secondary_category : [])],
+        pubDate: parseDate(article.show_time),
+    }));
 
     const out = await Promise.all(
-        list.map((item) => {
-            const title = item.title;
-            const link = `https://www.dongqiudi.com/news/${item.id}.html`;
-
-            return ctx.cache.tryGet(link, () => {
-                const single = {
-                    title,
-                    link,
-                };
-
-                const es = got(link);
-                proList.push(es);
-                return single;
-            });
-        })
-    );
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
 
-    const responses = await got.all(proList);
-    for (let i = 0; i < proList.length; i++) {
-        ProcessFeedType2(out[i], responses[i].data);
-    }
+                ProcessFeedType2(item, response);
+
+                return item;
+            })
+        )
+    );
 
     ctx.state.data = {
         title: `${name} - 相关新闻`,
         link,
+        image: type === 'team' ? typeInfo.team_logo : typeInfo.person_logo,
         item: out,
     };
 };
@@ -134,7 +134,7 @@ const ProcessFeedType2 = (item, response) => {
         ProcessImg(body('img'));
         item.description = body.html();
         item.author = data.writer;
-        item.pubDate = parseDate(data.show_time * 1000);
+        item.pubDate = parseDate(data.show_time, 'X');
     }
 };
 

package/lib/v2/douban/other/weekly_best.js

@@ -28,11 +28,13 @@ module.exports = async (ctx) => {
         link: `https://m.douban.com/subject_collection/${type}`,
         description: infoResponse.data.description,
 
-        item: data.map(({ title, info, cover, url, rating, year, release_date, null_rating_reason, description }) => {
+        item: data.map(({ title, info, cover, cover_url, url, rating, year, release_date, null_rating_reason, description }) => {
             const release = `${year}.${release_date}`;
             const rate = rating ? `${rating.value.toFixed(1)}分` : null_rating_reason;
-
-            description = `标题：${title} <br> 影片信息：${info} <br> 上映日期：${release} <br> 评分：${rate} <br> <img src="${cover.url}"> <p>${description}</p>`;
+            if (cover && cover.url) {
+                cover_url = cover.url;
+            }
+            description = `标题：${title} <br> 影片信息：${info} <br> 上映日期：${release} <br> 评分：${rate} <br> <img src="${cover_url}"> <p>${description}</p>`;
 
             return {
                 title,

package/lib/v2/douyin/hashtag.js

@@ -1,4 +1,3 @@
-const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const config = require('@/config').value;
@@ -23,19 +22,26 @@ module.exports = async (ctx) => {
             const browser = await require('@/utils/puppeteer')();
             const page = await browser.newPage();
             await page.setRequestInterception(true);
+            let awemeList = '';
             page.on('request', (request) => {
-                request.resourceType() === 'document' ? request.continue() : request.abort();
+                request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+            });
+            page.on('response', async (response) => {
+                const request = response.request();
+                if (request.url().includes('/v1/web/challenge/aweme')) {
+                    awemeList = await response.json();
+                }
             });
             await page.goto(tagUrl, {
-                waitUntil: 'domcontentloaded',
+                waitUntil: 'networkidle2',
             });
             await page.waitForSelector('#RENDER_DATA');
-            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            const html = await page.evaluate(() => document.getElementById('RENDER_DATA').innerText);
             await browser.close();
 
-            const $ = cheerio.load(html);
-            const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').text()));
-            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail && renderData[key].defaultData);
+            const renderData = JSON.parse(decodeURIComponent(html));
+            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail);
+            renderData[dataKey].defaultData = awemeList.aweme_list;
             return renderData[dataKey];
         },
         config.cache.routeExpire,
@@ -49,7 +55,7 @@ module.exports = async (ctx) => {
     const posts = tagData.defaultData;
     const items = posts.map((post) => {
         // parse video
-        let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
+        let videoList = post.video && post.video.bit_rate && post.video.bit_rate.map((bit_rate) => resolveUrl(bit_rate.play_addr.url_list[0]));
         if (relay) {
             videoList = videoList.map((item) => proxyVideo(item, relay));
         }
@@ -57,14 +63,9 @@ module.exports = async (ctx) => {
         duration = duration && duration / 1000;
         let img;
         // if (!embed) {
-        //     img = post.video && post.video.dynamicCover; // dynamic cover (webp)
+        //     img = post.video && post.video.dynamic_cover && post.video.dynamic_cover.url_list[post.video.dynamic_cover.url_list.length - 1]; // dynamic cover (webp)
         // }
-        img =
-            img ||
-            (post.video &&
-                ((post.video.coverUrlList && post.video.coverUrlList[post.video.coverUrlList.length - 1]) || // HD
-                    post.video.originCover || // LD
-                    post.video.cover)); // even more LD
+        img = img || (post.video && post.video.origin_cover && post.video.origin_cover.url_list[post.video.origin_cover.url_list.length - 1]);
         img = img && resolveUrl(img);
 
         // render description
@@ -76,10 +77,10 @@ module.exports = async (ctx) => {
         return {
             title: post.desc,
             description,
-            link: `https://www.douyin.com/video/${post.awemeId}`,
-            pubDate: parseDate(post.createTime * 1000),
-            category: post.textExtra.map((extra) => extra.hashtagName),
-            author: post.authorInfo.nickname,
+            link: `https://www.douyin.com/video/${post.aweme_id}`,
+            pubDate: parseDate(post.create_time, 'X'),
+            category: post.text_extra.map((extra) => extra.hashtag_name),
+            author: post.author.nickname,
         };
     });