rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/lib/v2/cw/master.js

@@ -1,29 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const { channel } = ctx.params;
-    const pageUrl = `${baseUrl}/masterChannel.action`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-        searchParams: {
-            idMasterChannel: channel,
-        },
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('master', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/masterChannel.action?idMasterChannel=${ctx.params.channel}`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/radar.js

@@ -20,6 +20,12 @@ module.exports = {
                 source: ['/subchannel.action'],
                 target: (_, url) => `/cw/sub/${new URL(url).searchParams.get('idSubChannel')}`,
             },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/traditional-media.html#tian-xia-za-zhi',
+                source: ['/author/:channel'],
+                target: '/cw/author/:channel',
+            },
         ],
     },
 };

package/lib/v2/cw/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
+    router.get('/author/:channel', require('./author'));
     router.get('/master/:channel', require('./master'));
     router.get('/sub/:channel', require('./sub'));
     router.get('/today', require('./today'));

package/lib/v2/cw/sub.js

@@ -1,29 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const { channel } = ctx.params;
-    const pageUrl = `${baseUrl}/subchannel.action`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-        searchParams: {
-            idSubChannel: channel,
-        },
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('sub', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/subchannel.action?idSubChannel=${ctx.params.channel}`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/today.js

@@ -1,25 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const pageUrl = `${baseUrl}/today`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('today', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/today`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/utils.js

@@ -1,26 +1,73 @@
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-const { Cookie, CookieJar } = require('tough-cookie');
-let cookieJar;
-const config = require('@/config').value;
+const { getCookies, setCookies } = require('@/utils/puppeteer-utils');
+const logger = require('@/utils/logger');
+let cookie;
 
 const baseUrl = 'https://www.cw.com.tw';
 
-const got = require('@/utils/got').extend({
-    headers: {
-        'User-Agent': config.trueUA,
+const pathMap = {
+    today: {
+        pageUrl: () => '/today',
+        limit: 30,
     },
-});
+    master: {
+        pageUrl: (channel) => `/masterChannel.action?idMasterChannel=${channel}`,
+        limit: 12,
+    },
+    sub: {
+        pageUrl: (channel) => `/subchannel.action?idSubChannel=${channel}`,
+        limit: 12,
+    },
+    author: {
+        pageUrl: (channel) => `/author/${channel}`,
+        limit: 10,
+    },
+};
 
-const getCookie = async () => {
-    const response = await got(`${baseUrl}/user/get/cookie-bar`);
-    const cookies = response.headers['set-cookie'];
-    if (Array.isArray(cookies)) {
-        cookieJar = cookies.map(Cookie.parse);
-    } else {
-        cookieJar = [Cookie.parse(cookieJar)];
+const getCookie = async (browser, tryGet) => {
+    if (!cookie) {
+        cookie = await tryGet('cw:cookie', async () => {
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
+            logger.debug(`Requesting ${baseUrl}/user/get/cookie-bar`);
+            await page.goto(`${baseUrl}/user/get/cookie-bar`, {
+                waitUntil: 'domcontentloaded',
+            });
+            cookie = await getCookies(page);
+            await page.close();
+            return cookie;
+        });
     }
-    cookieJar = CookieJar.fromJSON({ cookies: cookieJar });
+    return cookie;
+};
+
+const parsePage = async (path, browser, ctx) => {
+    const pageUrl = `${baseUrl}${pathMap[path].pageUrl(ctx.params.channel)}`;
+
+    const cookie = await getCookie(browser, ctx.cache.tryGet);
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await setCookies(page, cookie, 'cw.com.tw');
+    logger.debug(`Requesting ${pageUrl}`);
+    await page.goto(pageUrl, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    const response = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    const $ = cheerio.load(response);
+
+    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : pathMap[path].limit);
+    const items = await parseItems(list, browser, ctx.cache.tryGet);
+
+    return { $, items };
 };
 
 const parseList = ($, limit) =>
@@ -36,15 +83,25 @@ const parseList = ($, limit) =>
         })
         .slice(0, limit);
 
-const parseItems = (list, tryGet) =>
+const parseItems = (list, browser, tryGet) =>
     Promise.all(
         list.map((item) =>
             tryGet(item.link, async () => {
-                const { data: response } = await got(item.link, {
-                    cookieJar,
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await setCookies(page, cookie, 'cw.com.tw');
+                logger.debug(`Requesting ${item.link}`);
+                await page.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
                 });
 
+                const response = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
                 const $ = cheerio.load(response);
+
                 const meta = JSON.parse($('head script[type="application/ld+json"]').eq(0).text());
                 $('.article__head .breadcrumb, .article__head h1, .article__provideViews, .ad').remove();
                 $('img.lazyload').each((_, img) => {
@@ -67,9 +124,10 @@ const parseItems = (list, tryGet) =>
 
 module.exports = {
     baseUrl,
-    cookieJar,
-    got,
+    pathMap,
     getCookie,
+    setCookies,
+    parsePage,
     parseList,
     parseItems,
 };

package/lib/{routes → v2}/dlsite/campaign.js

@@ -139,31 +139,29 @@ module.exports = async (ctx) => {
     const title = `${info.name} | 割引中の作品`;
     const description = $('meta[name="description"]').attr('content');
     const list = $('tr[class]', '.n_worklist');
-    const item = list
-        .map((index, element) => {
-            const title = $('.work_name', element).text();
-            const link = $('.work_name > a', element).attr('href');
-            // 使链接
-            $('a', element).each((index, element) => {
-                $(element).attr('target', '_blank');
-            });
-            const description = $(element).html();
-            const arr = $('.search_tag', element);
-            const category = $('a', arr)
-                .map((index, a) => $(a).text())
-                .get();
-            const author = $('.maker_name', element).text();
+    const item = list.toArray().map((element) => {
+        const title = $('.work_name', element).text();
+        const link = $('.work_name > a', element).attr('href');
+        // 使链接
+        $('a', element).each((_index, element) => {
+            $(element).attr('target', '_blank');
+        });
+        const description = $(element).html();
+        const arr = $('.search_tag', element);
+        const category = $('a', arr)
+            .toArray()
+            .map((a) => $(a).text());
+        const author = $('.maker_name', element).text();
 
-            const signle = {
-                title,
-                link,
-                description,
-                category,
-                author,
-            };
-            return signle;
-        })
-        .get();
+        const signle = {
+            title,
+            link,
+            description,
+            category,
+            author,
+        };
+        return signle;
+    });
 
     ctx.state.data = {
         title,

package/lib/v2/dlsite/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/campaign/:type/:free?': ['cssxsh'],
+    '/new/:type': ['cssxsh'],
+};

package/lib/{routes → v2}/dlsite/new.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 const host = 'https://www.dlsite.com';
 const infos = {
@@ -69,35 +70,33 @@ module.exports = async (ctx) => {
     const list = $('.n_worklist_item');
     const dateText = $('.work_update')
         .text()
-        .replace(/(年 |月)/g, '-')
-        .substring(0, 10);
-    const pubDate = new Date(`${dateText} GMT+0900`).toUTCString();
-    const item = list
-        .map((index, element) => {
-            const title = $('.work_name', element).text();
-            const link = $('.work_name > a', element).attr('href');
-            // 使链接
-            $('a', element).each((index, element) => {
-                $(element).attr('target', '_blank');
-            });
-            const description = $(element).html();
-            const arr = $('.search_tag', element);
-            const category = $('a', arr)
-                .map((index, a) => $(a).text())
-                .get();
-            const author = $('.maker_name', element).text();
+        .trim()
+        .replace(/（.*）/g, '');
+    const pubDate = parseDate(dateText, 'YYYY年M月D日');
+    const item = list.toArray().map((element) => {
+        const title = $('.work_name', element).text();
+        const link = $('.work_name > a', element).attr('href');
+        // 使链接
+        $('a', element).each((_index, element) => {
+            $(element).attr('target', '_blank');
+        });
+        const description = $(element).html();
+        const arr = $('.search_tag', element);
+        const category = $('a', arr)
+            .toArray()
+            .map((a) => $(a).text());
+        const author = $('.maker_name', element).text();
 
-            const signle = {
-                title,
-                link,
-                description,
-                category,
-                author,
-                pubDate,
-            };
-            return signle;
-        })
-        .get();
+        const signle = {
+            title,
+            link,
+            description,
+            category,
+            author,
+            pubDate,
+        };
+        return signle;
+    });
 
     ctx.state.data = {
         title,

package/lib/v2/dlsite/radar.js

@@ -0,0 +1,15 @@
+module.exports = {
+    'dlsite.com': {
+        _name: 'DLsite',
+        '.': [
+            {
+                title: '当前日期发售的新产品',
+                docs: 'https://docs.rsshub.app/anime.html#dlsite',
+            },
+            {
+                title: '产品打折信息',
+                docs: 'https://docs.rsshub.app/anime.html#dlsite',
+            },
+        ],
+    },
+};

package/lib/v2/dlsite/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/campaign/:type/:free?', require('./campaign'));
+    router.get('/new/:type', require('./new'));
+};

package/lib/v2/dmzj/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:category?': ['vzz64'],
+};

package/lib/v2/dmzj/news.js

@@ -0,0 +1,26 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const url = `https://news.dmzj.com/${ctx.params.category || ''}`;
+    const $ = cheerio.load((await got(url)).data);
+    ctx.state.data = {
+        title: $('title').text(),
+        link: url,
+        item: $('.briefnews_con_li .li_img_de')
+            .map((_, item) => ({
+                title: $(item).find('h3 a').text(),
+                link: $(item).find('h3 a').attr('href'),
+                author: $(item).find('.head_con_p_o span:nth-child(3)').text().split('：')[1],
+                pubDate: timezone(parseDate($(item).find('.head_con_p_o span').first().text(), 'YYYY-MM-DD HH:mm'), +8),
+                description: $(item).find('p.com_about').text(),
+                category: $(item)
+                    .find('.u_comfoot a .bqwarp')
+                    .map((_, item) => $(item).text())
+                    .get(),
+            }))
+            .get(),
+    };
+};

package/lib/v2/dmzj/radar.js

@@ -0,0 +1,67 @@
+module.exports = {
+    'dmzj.com': {
+        _name: '动漫之家',
+        news: [
+            {
+                title: '宅新闻',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/',
+                target: '/dmzj/news',
+            },
+            {
+                title: '漫画情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manhuaqingbao',
+                target: '/dmzj/news/manhuaqingbao',
+            },
+            {
+                title: '轻小说情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/qingxiaoshuoqingbao',
+                target: '/dmzj/news/qingxiaoshuoqingbao',
+            },
+            {
+                title: '动漫周边',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manhuazhoubian',
+                target: '/dmzj/news/manhuazhoubian',
+            },
+            {
+                title: '声优情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/shengyouqingbao',
+                target: '/dmzj/news/shengyouqingbao',
+            },
+            {
+                title: '音乐资讯',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/yinyuezixun',
+                target: '/dmzj/news/yinyuezixun',
+            },
+            {
+                title: '游戏资讯',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/youxizixun',
+                target: '/dmzj/news/youxizixun',
+            },
+            {
+                title: '美图欣赏',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/meituxinshang',
+                target: '/dmzj/news/meituxinshang',
+            },
+            {
+                title: '漫展情报',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/manzhanqingbao',
+                target: '/dmzj/news/manzhanqingbao',
+            },
+            {
+                title: '大杂烩',
+                docs: 'https://docs.rsshub.app/anime.html#dong-man-zhi-jia',
+                source: '/dazahui',
+                target: '/dmzj/news/dazahui',
+            },
+        ],
+    },
+};

package/lib/v2/dmzj/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/news/:category?', require('./news'));
+};

package/lib/v2/douban/other/weekly_best.js

@@ -1,4 +1,6 @@
 const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
     const type = ctx.params.type || 'movie_weekly_best';
@@ -28,17 +30,22 @@ module.exports = async (ctx) => {
         link: `https://m.douban.com/subject_collection/${type}`,
         description: infoResponse.data.description,
 
-        item: data.map(({ title, info, cover, cover_url, url, rating, year, release_date, null_rating_reason, description }) => {
-            const release = `${year}.${release_date}`;
+        item: data.map(({ title, cover, cover_url, url, rating, null_rating_reason, description, card_subtitle, photos }) => {
             const rate = rating ? `${rating.value.toFixed(1)}分` : null_rating_reason;
             if (cover && cover.url) {
                 cover_url = cover.url;
             }
-            description = `标题：${title} <br> 影片信息：${info} <br> 上映日期：${release} <br> 评分：${rate} <br> <img src="${cover_url}"> <p>${description}</p>`;
 
             return {
                 title,
-                description,
+                description: art(path.join(__dirname, '../templates/weekly_best.art'), {
+                    title,
+                    card_subtitle,
+                    description,
+                    rate,
+                    cover_url,
+                    photos,
+                }),
                 link: url,
             };
         }),

package/lib/v2/douban/templates/weekly_best.art

@@ -0,0 +1,10 @@
+<p>标题：{{title}}</p>
+<p>评分：{{rate}}</p>
+<p>标签：{{card_subtitle}}</p>
+<p>影片信息：{{description}}</p>
+<p>
+    <img src="{{cover_url}}" />
+    {{each photos}}
+        <img src="{{$value}}" />
+    {{/each}}
+</p>

package/lib/v2/dut/index.js

@@ -3,9 +3,13 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const defaults = require('./defaults');
 const shortcuts = require('./shortcuts');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const site = ctx.params[0] ?? 'news';
+    if (!isValidHost(site)) {
+        throw Error('Invalid site');
+    }
 
     let items;
     let category = ctx.params[1] ?? (defaults.hasOwnProperty(site) ? defaults[site] : '');

package/lib/v2/eagle/blog.js

@@ -1,16 +1,20 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-
+const { isValidHost } = require('@/utils/valid-host');
 const cateList = ['all', 'design-resources', 'learn-design', 'inside-eagle'];
 
 module.exports = async (ctx) => {
     let cate = ctx.params.cate ?? 'all';
     let language = ctx.params.language ?? 'cn';
-    if (cateList.indexOf(cate) === -1) {
+    if (!isValidHost(cate) || !isValidHost(language)) {
+        throw Error('Invalid host');
+    }
+    if (!cateList.includes(cate)) {
         language = cate;
         cate = 'all';
     }
+
     const host = `https://${language}.eagle.cool`;
     const url = `${host}/blog/${cate === 'all' ? '' : cate}`;