rsshub 1.0.0-master.fdbb6fa → 1.0.0-master.fdc59d2

package/lib/v2/domp4/detail.js

@@ -1,6 +1,6 @@
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
-const { decodeCipherText, composeMagnetUrl } = require('./utils');
+const { decodeCipherText, composeMagnetUrl, getUrlType } = require('./utils');
 
 const baseUrl = 'https://www.domp4.cc';
 
@@ -16,13 +16,17 @@ function getItemList($, detailUrl) {
     );
     const { downurls } = data.Data[0];
     return downurls.map((item) => {
-        const [title, magnet] = item.split('$');
+        const [title, downurl] = item.split('$');
+        const urlType = getUrlType(downurl);
+        // only magnet need compose trackers
+        const enclosureUrl = urlType === 'magnet' ? composeMagnetUrl(downurl) : downurl;
         return {
-            enclosure_url: composeMagnetUrl(magnet),
+            enclosure_url: enclosureUrl,
             enclosure_length: '',
             enclosure_type: 'application/x-bittorrent',
             title,
             link: detailUrl,
+            guid: `${title}-${urlType}`,
         };
     });
 }

package/lib/v2/domp4/utils.js

@@ -35,6 +35,19 @@ function composeMagnetUrl(magnet, trackers = magnetTrackers) {
     return `${magnet}&tr=${trackers.join('&tr=')}`;
 }
 
+/**
+ * method for download url type
+ */
+function getUrlType(url) {
+    if (url.startsWith('magnet:')) {
+        return 'magnet';
+    }
+    if (url.startsWith('ed2k:')) {
+        return 'ed2k';
+    }
+    return '';
+}
+
 /**
  * method for generating magnet url
  * from the detail page source
@@ -67,6 +80,7 @@ function decodeCipherText(p, a, c, k, e, d) {
 
 module.exports = {
     magnetTrackers,
+    getUrlType,
     composeMagnetUrl,
     decodeCipherText,
 };

package/lib/v2/dongqiudi/player_news.js

@@ -1,8 +1,7 @@
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
-    const id = ctx.params.id;
-    const link = `https://www.dongqiudi.com/player/${id}.html`;
+    const playerId = ctx.params.id;
 
-    await utils.ProcessFeed(ctx, link, 'player');
+    await utils.ProcessFeed(ctx, 'player', playerId);
 };

package/lib/v2/dongqiudi/result.js

@@ -6,7 +6,7 @@ module.exports = async (ctx) => {
     const team = ctx.params.team;
     const link = `https://www.dongqiudi.com/team/${team}.html`;
 
-    const response = await got.get(link);
+    const response = await got(link);
     const dom = new JSDOM(response.data, {
         runScripts: 'dangerously',
     });

package/lib/v2/dongqiudi/router.js

@@ -1,5 +1,5 @@
 module.exports = (router) => {
-    router.get('/daily', require('./daily'));
+    router.redirect('/daily', '/dongqiudi/special/48');
     router.get('/player_news/:id', require('./player_news'));
     router.get('/result/:team', require('./result'));
     router.get('/special/:id', require('./special'));

package/lib/v2/dongqiudi/special.js

@@ -4,44 +4,38 @@ const utils = require('./utils');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
-    const response = await got.get(`https://www.dongqiudi.com/special/${id}`);
+    const response = await got(`https://www.dongqiudi.com/special/${id}`);
 
     const $ = cheerio.load(response.data);
 
     const host = 'https://www.dongqiudi.com';
 
-    const list = $('.detail.special ul li h3').slice(0, 5).get();
-
-    const proList = [];
+    const list = $('.detail.special ul li h3')
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 5)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').text(),
+                link: host + item.find('a').attr('href'),
+            };
+        });
 
     const out = await Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const title = $('a').text();
-            const itemUrl = host + $('a').attr('href');
-
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const single = {
-                title,
-                link: itemUrl,
-            };
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+
+                utils.ProcessFeedType2(item, response);
 
-            const es = got.get(itemUrl);
-            proList.push(es);
-            return Promise.resolve(single);
-        })
+                return item;
+            })
+        )
     );
 
-    const responses = await got.all(proList);
-    for (let i = 0; i < proList.length; i++) {
-        utils.ProcessFeedType2(out[i], responses[i].data);
-    }
     ctx.state.data = {
-        title: `懂球帝专题-${id}`,
+        title: `懂球帝专题-${$('.detail.special h1').text()}`,
+        description: $('.detail.special h4').text(),
         link: `https://www.dongqiudi.com/special/${id}`,
         item: out.filter((e) => e !== undefined),
     };

package/lib/v2/dongqiudi/team_news.js

@@ -1,8 +1,7 @@
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
-    const team = ctx.params.team;
-    const link = `https://www.dongqiudi.com/team/${team}.html`;
+    const teamId = ctx.params.team;
 
-    await utils.ProcessFeed(ctx, link, 'team');
+    await utils.ProcessFeed(ctx, 'team', teamId);
 };

package/lib/v2/dongqiudi/top_news.js

@@ -1,39 +1,33 @@
 const got = require('@/utils/got');
 const utils = require('./utils');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id ?? 1;
-    const response = await got(`https://api.dongqiudi.com/app/tabs/iphone/${id}.json?mark=gif&version=576`);
-    const data = response.data.articles;
-    const label = response.data.label;
+    const { data } = await got(`https://api.dongqiudi.com/app/tabs/web/${id}.json`);
+    const articles = data.articles;
+    const label = data.label;
 
-    const proList = [];
+    const list = articles.map((item) => ({
+        title: item.title,
+        link: `https://www.dongqiudi.com/articles/${item.id}.html`,
+        category: [item.category, ...(item.secondary_category ? item.secondary_category : [])],
+        pubDate: parseDate(item.show_time),
+    }));
 
     const out = await Promise.all(
-        data.map((item) => {
-            const title = item.title;
-            const itemUrl = `https://www.dongqiudi.com/news/${item.id}.html`;
-
-            return ctx.cache.tryGet(itemUrl, () => {
-                const single = {
-                    title,
-                    link: itemUrl,
-                };
-
-                const es = got(itemUrl);
-                proList.push(es);
-                return single;
-            });
-        })
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                utils.ProcessFeedType2(item, response);
+                return item;
+            })
+        )
     );
 
-    const responses = await got.all(proList);
-    for (let i = 0; i < proList.length; i++) {
-        utils.ProcessFeedType2(out[i], responses[i].data);
-    }
     ctx.state.data = {
         title: `懂球帝 - ${label}`,
-        link: 'http://dongqiudi.com/news',
+        link: `https://www.dongqiudi.com/articlesList/${id}`,
         item: out.filter((e) => e !== undefined),
     };
 };

package/lib/v2/dongqiudi/utils.js

@@ -61,57 +61,57 @@ const ProcessImg = (content) => {
     });
 };
 
-const ProcessFeed = async (ctx, link, type) => {
-    const got_ins = got.extend({
-        headers: {
-            Referer: link,
-        },
-    });
-
-    const response = await got_ins.get(link);
+const ProcessFeed = async (ctx, type, id) => {
+    const link = `https://www.dongqiudi.com/${type}/${id}.html`;
+    const apiUrl = `https://api.dongqiudi.com/v3/archive/app/channel/feeds`;
+    const { data: response } = await got(link);
 
     let name;
 
-    const dom = new JSDOM(response.data, {
+    const { window } = new JSDOM(response, {
         runScripts: 'dangerously',
     });
 
-    const data = dom.window.__NUXT__.data[0];
-    const list = data.relatedNewsList;
-
+    const typeInfo = window.__NUXT__.data[0][`${type}Detail`].base_info;
     if (type === 'team') {
-        name = data.teamDetail.base_info.team_name;
+        name = typeInfo.team_name;
     } else if (type === 'player') {
-        name = data.playerDetail.base_info.person_name;
+        name = typeInfo.person_name;
     }
-    const proList = [];
+
+    const { data } = await got(apiUrl, {
+        searchParams: {
+            id,
+            type,
+            size: 20,
+            platform: 'web',
+            version: '',
+        },
+    });
+
+    const list = data.data.articles.map((article) => ({
+        title: article.title,
+        link: `https://www.dongqiudi.com/articles/${article.id}.html`,
+        category: [article.category, ...(article.secondary_category ? article.secondary_category : [])],
+        pubDate: parseDate(article.show_time),
+    }));
 
     const out = await Promise.all(
-        list.map((item) => {
-            const title = item.title;
-            const link = `https://www.dongqiudi.com/news/${item.id}.html`;
-
-            return ctx.cache.tryGet(link, () => {
-                const single = {
-                    title,
-                    link,
-                };
-
-                const es = got(link);
-                proList.push(es);
-                return single;
-            });
-        })
-    );
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
 
-    const responses = await got.all(proList);
-    for (let i = 0; i < proList.length; i++) {
-        ProcessFeedType2(out[i], responses[i].data);
-    }
+                ProcessFeedType2(item, response);
+
+                return item;
+            })
+        )
+    );
 
     ctx.state.data = {
         title: `${name} - 相关新闻`,
         link,
+        image: type === 'team' ? typeInfo.team_logo : typeInfo.person_logo,
         item: out,
     };
 };
@@ -134,7 +134,7 @@ const ProcessFeedType2 = (item, response) => {
         ProcessImg(body('img'));
         item.description = body.html();
         item.author = data.writer;
-        item.pubDate = parseDate(data.show_time * 1000);
+        item.pubDate = parseDate(data.show_time, 'X');
     }
 };
 

package/lib/v2/douban/other/weekly_best.js

@@ -28,11 +28,13 @@ module.exports = async (ctx) => {
         link: `https://m.douban.com/subject_collection/${type}`,
         description: infoResponse.data.description,
 
-        item: data.map(({ title, info, cover, url, rating, year, release_date, null_rating_reason, description }) => {
+        item: data.map(({ title, info, cover, cover_url, url, rating, year, release_date, null_rating_reason, description }) => {
             const release = `${year}.${release_date}`;
             const rate = rating ? `${rating.value.toFixed(1)}分` : null_rating_reason;
-
-            description = `标题：${title} <br> 影片信息：${info} <br> 上映日期：${release} <br> 评分：${rate} <br> <img src="${cover.url}"> <p>${description}</p>`;
+            if (cover && cover.url) {
+                cover_url = cover.url;
+            }
+            description = `标题：${title} <br> 影片信息：${info} <br> 上映日期：${release} <br> 评分：${rate} <br> <img src="${cover_url}"> <p>${description}</p>`;
 
             return {
                 title,

package/lib/v2/douyin/live.js

@@ -21,7 +21,7 @@ module.exports = async (ctx) => {
             await page.goto(pageUrl, {
                 waitUntil: 'domcontentloaded',
             });
-            const data = await page.evaluate(() => renderData);
+            const data = JSON.parse(decodeURIComponent(await page.evaluate(() => document.querySelector('#RENDER_DATA').innerText)));
             browser.close();
 
             return data;
@@ -30,11 +30,11 @@ module.exports = async (ctx) => {
         false
     );
 
-    if (renderData.routeInitialProps.status_code !== '0') {
-        throw `Status code ${renderData.routeInitialProps.status_code}: ${renderData.routeInitialProps.error.title}`;
+    if (renderData[Object.keys(renderData).find((k) => renderData[k].status_code)].status_code !== '0') {
+        throw `Status code ${renderData[Object.keys(renderData).find((k) => renderData[k].status_code)].status_code}`;
     }
 
-    const roomInfo = renderData.initialState.roomStore.roomInfo;
+    const roomInfo = renderData.app.initialState.roomStore.roomInfo;
     const nickname = roomInfo.anchor.nickname;
     const userAvatar = roomInfo.anchor.avatar_thumb.url_list[0];
 

package/lib/v2/douyu/group.js

@@ -0,0 +1,45 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+    const sort = ctx.params.sort ?? '2';
+
+    const rootUrl = 'https://yuba.douyu.com';
+    const detailUrl = `${rootUrl}/wbapi/web/group/head?group_id=${id}`;
+    const apiUrl = `${rootUrl}/wbapi/web/group/postlist?group_id=${id}&page=1&sort=${sort}`;
+    const currentUrl = `${rootUrl}/group/${sort === '1' ? 'newall' : 'newself'}/${id}`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    const detailResponse = await got({
+        method: 'get',
+        url: detailUrl,
+    });
+
+    const items = response.data.data.map((item) => ({
+        title: item.title,
+        link: `${rootUrl}/p/${item.post_id}`,
+        pubDate: timezone(parseDate(item.created_at_std), +8),
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            content: item.describe,
+            images: item.imglist.map((i) => ({
+                size: i.size,
+                url: i.url,
+            })),
+        }),
+    }));
+
+    ctx.state.data = {
+        title: `斗鱼鱼吧 - ${detailResponse.data.data.group_name}`,
+        link: currentUrl,
+        item: items,
+        description: detailResponse.data.data.describe,
+    };
+};

package/lib/v2/douyu/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/group/:id/:sort?': ['nczitzk'],
+    '/post/:id': ['nczitzk'],
+    '/room/:id': ['DIYgod'],
+};

package/lib/v2/douyu/post.js

@@ -0,0 +1,58 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const rootUrl = 'https://yuba.douyu.com';
+    const currentUrl = `${rootUrl}/p/${id}`;
+    const detailUrl = `${rootUrl}/wbapi/web/post/detail/${id}`;
+
+    const detailResponse = await got({
+        method: 'get',
+        url: detailUrl,
+    });
+
+    const data = detailResponse.data.data;
+    let apiUrl = `${rootUrl}/wbapi/web/post/comments/${id}?group_id=${data.group_id}&sink=1&page=1`;
+
+    let response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    apiUrl = `${rootUrl}/wbapi/web/post/comments/${id}?group_id=${data.group_id}&sink=1&page=${response.data.page_total}`;
+
+    response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    const items = response.data.data.map((item) => ({
+        title: `${item.nick_name}: ${item.content}`,
+        link: `${currentUrl}#${item.comment_id}${item.sub_replies.length > 0 ? `+${item.sub_replies.map((r) => r.comment_id).join('+')}` : ''}`,
+        pubDate: parseDate(item.created_ts * 1000),
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            content: item.content,
+            images: item.imglist.map((i) => ({
+                size: i.size,
+                url: i.url,
+            })),
+            replies:
+                item.sub_replies.map((r) => ({
+                    nickname: r.nickname,
+                    content: r.content,
+                    time: new Date(r.created_ts * 1000).toLocaleString(),
+                })) ?? undefined,
+        }),
+    }));
+
+    ctx.state.data = {
+        title: `斗鱼鱼吧 - ${data.title}`,
+        link: currentUrl,
+        item: items,
+        description: data.content,
+    };
+};

package/lib/v2/douyu/radar.js

@@ -0,0 +1,27 @@
+module.exports = {
+    'douyu.com': {
+        _name: '斗鱼',
+        www: [
+            {
+                title: '直播间开播',
+                docs: 'https://docs.rsshub.app/live.html#dou-yu-zhi-bo-zhi-bo-jian-kai-bo',
+                source: ['/:id', '/'],
+                target: '/douyu/room/:id',
+            },
+        ],
+        yuba: [
+            {
+                title: '鱼吧帖子',
+                docs: 'https://docs.rsshub.app/bbs.html#dou-yu-yu-ba-tie-zi',
+                source: ['/group/:id', '/group/newself/:id', '/group/newall/:id', '/'],
+                target: '/douyu/group/:id',
+            },
+            {
+                title: '鱼吧跟帖',
+                docs: 'https://docs.rsshub.app/bbs.html#dou-yu-yu-ba-gen-tie',
+                source: ['/p/:id', '/'],
+                target: '/douyu/post/:id',
+            },
+        ],
+    },
+};

package/lib/v2/douyu/router.js

@@ -0,0 +1,5 @@
+module.exports = function (router) {
+    router.get('/group/:id/:sort?', require('./group'));
+    router.get('/post/:id', require('./post'));
+    router.get('/room/:id', require('./room'));
+};

package/lib/v2/douyu/templates/description.art

@@ -0,0 +1,14 @@
+{{ if content }}
+{{ content }}
+{{ /if }}
+{{ if images }}
+{{ each images image }}
+<img src="{{ image.url }}" width="{{ image.size.w }}" height="{{ image.size.h }}">
+{{ /each }}
+{{ /if }}
+{{ if replies }}
+<h3>回复</h3>
+{{ each replies reply }}
+<p><b>{{ reply.nickname }}</b> <small>[{{ reply.time }}]</small>: {{ reply.content }}</p>
+{{ /each }}
+{{ /if }}

package/lib/v2/economist/full.js

@@ -7,7 +7,7 @@ const getArticleDetail = async (link, ctx) =>
         const response = await got(link);
         const $ = cheerio.load(response.data);
         $('div.article-audio-player__center-tooltip').remove();
-        const nextData = JSON.parse($('head script[type="application/ld+json"]').text());
+        const nextData = JSON.parse($('head script[type="application/ld+json"]').first().text());
 
         const article = ($('figure[class^=css-]').first().parent().parent().html() || '') + $('p.article__body-text').parent().html();
         const categories = nextData.keywords?.map((k) => k);

package/lib/v2/ehentai/ehapi.js

@@ -17,6 +17,10 @@ if (has_cookie) {
     }
 }
 
+if (config.star) {
+    headers.cookie += `;star=${config.star}`;
+}
+
 function ehgot(url) {
     if (from_ex) {
         return got({ method: 'get', url: `https://exhentai.org/${url}`, headers });
@@ -41,22 +45,41 @@ function ehgot_thumb(cache, thumb_url) {
 
 async function parsePage(cache, data, get_bittorrent = false, embed_thumb = false) {
     const $ = cheerio.load(data);
-    const table = $('table[class^="itg glt"] tbody');
-    if (!table) {
-        return [];
+    // "m" for Minimal
+    // "p" for Minimal+
+    // "l" for Compact
+    // "e" for Extended
+    // "t" for Thumbnail
+    const layout = $('div#dms option[selected]').attr('value');
+
+    let galleries;
+    if (layout !== 't') {
+        galleries = $('table[class^="itg glt"] tbody');
+    } else {
+        // Thumbnail layout has different page structure
+        // resolve it here
+        galleries = $('div[class^="itg gld"]');
     }
 
-    async function parseTableRow(cache, element) {
+    async function parseElement(cache, element) {
         const el = $(element);
         const title = el.find('div.glink').html();
         const pubDate = el.find('div[id^="posted_"]').html();
-        // match "Minimal", "Minimal+" and "Compact"
-        let el_a = el.find('td[class^="gl3"] a');
-        let el_img = el.find('td[class^=gl2] div.glthumb div img');
-        // if not, match "Extended"
-        if (!(el_a.length && el_img.length)) {
+        let el_a;
+        let el_img;
+        // match layout
+        if ('mpl'.includes(layout)) {
+            // Minimal, Minimal+, Compact
+            el_a = el.find('td[class^="gl3"] a');
+            el_img = el.find('td[class^=gl2] div.glthumb div img');
+        } else if (layout === 'e') {
+            // Extended
             el_a = el.find('td[class^="gl1"] a');
             el_img = el_a.find('img');
+        } else if (layout === 't') {
+            // Thumbnail
+            el_a = el.find('div[class^="gl3t"] a');
+            el_img = el_a.find('img');
         }
         const link = el_a.attr('href');
         let thumbnail = el_img.attr('data-src') ? el_img.attr('data-src') : el_img.attr('src');
@@ -82,13 +105,23 @@ async function parsePage(cache, data, get_bittorrent = false, embed_thumb = fals
                     }
                 }
             }
+            if ('le'.includes(layout)) {
+                // artist tags will only show in Compact or Extended layout
+                // get artist names as author
+                const artist = $(el)
+                    .find('div.gt[title^="artist:"]')
+                    .toArray()
+                    .map((i, tag) => $(tag).text())
+                    .join(' / ');
+                item.author = artist;
+            }
             return item;
         }
     }
 
     const item_Promises = [];
-    table.children('tr').each((index, element) => {
-        item_Promises.push(parseTableRow(cache, element));
+    galleries.children().each((index, element) => {
+        item_Promises.push(parseElement(cache, element));
     });
     const items_with_null = await Promise.all(item_Promises);
 

package/lib/v2/ehentai/maintainer.js

@@ -1,5 +1,5 @@
 module.exports = {
-    '/favorites/:favcat?/:order?/:page?/:routeParams?': ['yindaheng98'],
-    '/search/:params?/:page?/:routeParams?': ['yindaheng98'],
-    '/tag/:tag/:page?/:routeParams?': ['yindaheng98'],
+    '/favorites/:favcat?/:order?/:page?/:routeParams?': ['yindaheng98', 'syrinka'],
+    '/search/:params?/:page?/:routeParams?': ['yindaheng98', 'syrinka'],
+    '/tag/:tag/:page?/:routeParams?': ['yindaheng98', 'syrinka'],
 };

package/lib/v2/fantia/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/search/:type?/:caty?/:peroid?/:order?/:rating?/:keyword?': ['nczitzk'],
+    '/user/:id': ['nczitzk'],
+};

package/lib/v2/fantia/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'fantia.jp': {
+        _name: 'Fantia',
+        '.': [
+            {
+                title: '用户投稿',
+                docs: 'https://docs.rsshub.app/picture.html#fantia',
+                source: ['/fanclubs/:id'],
+                target: '/fantia/user/:id',
+            },
+        ],
+    },
+};