rsshub 1.0.0-master.f961961 → 1.0.0-master.f992838

package/lib/v2/copymanga/comic.js

@@ -0,0 +1,110 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const host = 'copymanga.site';
+const baseUrl = `https://${host}`;
+const apiBaseUrl = `https://api.${host}`;
+
+// 直接调用拷贝漫画的接口
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    // 用于控制返回的章节数量
+    const chapterCnt = Number(ctx.params.chapterCnt || 0);
+    // 获取漫画列表
+    let bHasNextPage = false;
+    const iReqLimit = (ctx.queries && ctx.queries.limit) || 100;
+    let iReqOffSet = 0;
+    const strBaseUrl = `${apiBaseUrl}/api/v3/comic/${id}/group/default/chapters`;
+    let chapterArray = [];
+
+    do {
+        bHasNextPage = false;
+        // eslint-disable-next-line no-await-in-loop
+        const { data } = await got(strBaseUrl, {
+            headers: {
+                platform: 1,
+            },
+            searchParams: {
+                limit: iReqLimit,
+                offset: iReqOffSet === 0 ? undefined : iReqOffSet,
+            },
+        });
+        const { code, results } = data;
+
+        if (code !== 200) {
+            break;
+        }
+
+        if (results.limit + results.offset < results.total) {
+            bHasNextPage = true;
+        }
+        iReqOffSet += iReqLimit;
+
+        chapterArray = [...chapterArray, ...results.list];
+    } while (bHasNextPage);
+
+    chapterArray = chapterArray
+        .map(({ comic_path_word, uuid, name, size, datetime_created /* , index*/ }) => ({
+            link: `${baseUrl}/comic/${comic_path_word}/chapter/${uuid}`,
+            uuid,
+            title: name,
+            size,
+            pubDate: datetime_created,
+            // index,
+        }))
+        .reverse();
+
+    // 获取漫画标题、介绍
+    const { bookTitle, bookIntro } = await ctx.cache.tryGet(`${baseUrl}/comic/${id}`, async () => {
+        const { data } = await got(`${baseUrl}/comic/${id}`);
+        const $ = cheerio.load(data);
+        return {
+            bookTitle: $('.comicParticulars-title-right > ul > li > h6').text(),
+            bookIntro: $('.intro').text(),
+        };
+    });
+
+    const genResult = async (chapter) => {
+        const {
+            data: { results },
+        } = await got(`${apiBaseUrl}/api/v3/comic/${id}/chapter2/${chapter.uuid}`, {
+            headers: {
+                webp: 1,
+            },
+        });
+
+        const pageNumbers = Object.values(results.chapter.words);
+        const contents = results.chapter.contents
+            .map((content, index) => ({
+                page: pageNumbers[index],
+                url: content.url,
+            }))
+            .sort((a, b) => a.page - b.page);
+
+        return {
+            link: chapter.link,
+            title: chapter.title,
+            description: art(path.join(__dirname, './templates/comic.art'), {
+                size: chapter.size,
+                contents,
+            }),
+            pubDate: parseDate(chapter.pubDate, 'YYYY-MM-DD'),
+        };
+    };
+
+    let itemsLen = chapterArray.length;
+
+    if (chapterCnt > 0) {
+        itemsLen = chapterCnt;
+    }
+
+    ctx.state.data = {
+        title: `拷贝漫画 - ${bookTitle}`,
+        link: `${baseUrl}/comic/${id}`,
+        description: bookIntro,
+        item: await Promise.all(chapterArray.slice(0, itemsLen).map((chapter) => ctx.cache.tryGet(chapter.link, () => genResult(chapter)))),
+    };
+};

package/lib/v2/copymanga/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/comic/:id/:chapterCnt?': ['btdwv', 'marvolo666', 'yan12125'],
+};

package/lib/v2/copymanga/radar.js

@@ -0,0 +1,19 @@
+const copymanga = {
+    _name: '拷贝漫画',
+    '.': [
+        {
+            title: '漫画更新',
+            docs: 'https://docs.rsshub.app/anime.html#kao-bei-man-hua',
+            source: '/comic/:id',
+            target: '/copymanga/comic/:id/5',
+        },
+    ],
+};
+
+module.exports = {
+    'copymanga.com': copymanga,
+    'copymanga.info': copymanga,
+    'copymanga.net': copymanga,
+    'copymanga.org': copymanga,
+    'copymanga.site': copymanga,
+};

package/lib/v2/copymanga/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/comic/:id/:chapterCnt?', require('./comic'));
+};

package/lib/v2/copymanga/templates/comic.art

@@ -0,0 +1,4 @@
+<h1>{{ size }}p</h1><br>
+{{ each contents img }}
+<img src="{{ img.url }}" alt="{{ img.page }}">
+{{ /each }}

package/lib/v2/cw/author.js

@@ -0,0 +1,18 @@
+const { baseUrl, parsePage } = require('./utils');
+
+module.exports = async (ctx) => {
+    const browser = await require('@/utils/puppeteer')();
+
+    const { $, items } = await parsePage('author', browser, ctx);
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('.authorTxt').text(),
+        link: `${baseUrl}/author/${ctx.params.channel}`,
+        image: $('.authorPhoto img').attr('src') || `${baseUrl}/assets_new/img/fbshare.jpg'`,
+        language: $('meta[property="og:locale"]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/cw/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/author/:channel': ['TonyRL'],
     '/master/:channel': ['TonyRL'],
     '/sub/:channel': ['TonyRL'],
     '/today': ['TonyRL'],

package/lib/v2/cw/master.js

@@ -1,29 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const { channel } = ctx.params;
-    const pageUrl = `${baseUrl}/masterChannel.action`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-        searchParams: {
-            idMasterChannel: channel,
-        },
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('master', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/masterChannel.action?idMasterChannel=${ctx.params.channel}`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/radar.js

@@ -20,6 +20,12 @@ module.exports = {
                 source: ['/subchannel.action'],
                 target: (_, url) => `/cw/sub/${new URL(url).searchParams.get('idSubChannel')}`,
             },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/traditional-media.html#tian-xia-za-zhi',
+                source: ['/author/:channel'],
+                target: '/cw/author/:channel',
+            },
         ],
     },
 };

package/lib/v2/cw/router.js

@@ -1,4 +1,5 @@
 module.exports = (router) => {
+    router.get('/author/:channel', require('./author'));
     router.get('/master/:channel', require('./master'));
     router.get('/sub/:channel', require('./sub'));
     router.get('/today', require('./today'));

package/lib/v2/cw/sub.js

@@ -1,29 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const { channel } = ctx.params;
-    const pageUrl = `${baseUrl}/subchannel.action`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-        searchParams: {
-            idSubChannel: channel,
-        },
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('sub', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/subchannel.action?idSubChannel=${ctx.params.channel}`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/today.js

@@ -1,25 +1,16 @@
-const cheerio = require('cheerio');
-const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+const { baseUrl, parsePage } = require('./utils');
 
 module.exports = async (ctx) => {
-    const pageUrl = `${baseUrl}/today`;
-    if (!cookieJar) {
-        await getCookie();
-    }
-    const { data: response } = await got(pageUrl, {
-        headers: {
-            Referer: baseUrl,
-        },
-        cookieJar,
-    });
-    const $ = cheerio.load(response);
+    const browser = await require('@/utils/puppeteer')();
 
-    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
-    const items = await parseItems(list, ctx.cache.tryGet);
+    const { $, items } = await parsePage('today', browser, ctx);
+
+    await browser.close();
 
     ctx.state.data = {
         title: $('head title').text(),
         description: $('meta[name=description]').attr('content'),
+        link: `${baseUrl}/today`,
         image: `${baseUrl}/assets_new/img/fbshare.jpg`,
         language: $('meta[property="og:locale"]').attr('content'),
         item: items,

package/lib/v2/cw/utils.js

@@ -1,26 +1,73 @@
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
-const { Cookie, CookieJar } = require('tough-cookie');
-let cookieJar;
-const config = require('@/config').value;
+const { getCookies, setCookies } = require('@/utils/puppeteer-utils');
+const logger = require('@/utils/logger');
+let cookie;
 
 const baseUrl = 'https://www.cw.com.tw';
 
-const got = require('@/utils/got').extend({
-    headers: {
-        'User-Agent': config.trueUA,
+const pathMap = {
+    today: {
+        pageUrl: () => '/today',
+        limit: 30,
     },
-});
+    master: {
+        pageUrl: (channel) => `/masterChannel.action?idMasterChannel=${channel}`,
+        limit: 12,
+    },
+    sub: {
+        pageUrl: (channel) => `/subchannel.action?idSubChannel=${channel}`,
+        limit: 12,
+    },
+    author: {
+        pageUrl: (channel) => `/author/${channel}`,
+        limit: 10,
+    },
+};
 
-const getCookie = async () => {
-    const response = await got(`${baseUrl}/user/get/cookie-bar`);
-    const cookies = response.headers['set-cookie'];
-    if (Array.isArray(cookies)) {
-        cookieJar = cookies.map(Cookie.parse);
-    } else {
-        cookieJar = [Cookie.parse(cookieJar)];
+const getCookie = async (browser, tryGet) => {
+    if (!cookie) {
+        cookie = await tryGet('cw:cookie', async () => {
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
+            logger.debug(`Requesting ${baseUrl}/user/get/cookie-bar`);
+            await page.goto(`${baseUrl}/user/get/cookie-bar`, {
+                waitUntil: 'domcontentloaded',
+            });
+            cookie = await getCookies(page);
+            await page.close();
+            return cookie;
+        });
     }
-    cookieJar = CookieJar.fromJSON({ cookies: cookieJar });
+    return cookie;
+};
+
+const parsePage = async (path, browser, ctx) => {
+    const pageUrl = `${baseUrl}${pathMap[path].pageUrl(ctx.params.channel)}`;
+
+    const cookie = await getCookie(browser, ctx.cache.tryGet);
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await setCookies(page, cookie, 'cw.com.tw');
+    logger.debug(`Requesting ${pageUrl}`);
+    await page.goto(pageUrl, {
+        waitUntil: 'domcontentloaded',
+    });
+
+    const response = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+    const $ = cheerio.load(response);
+
+    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : pathMap[path].limit);
+    const items = await parseItems(list, browser, ctx.cache.tryGet);
+
+    return { $, items };
 };
 
 const parseList = ($, limit) =>
@@ -36,15 +83,25 @@ const parseList = ($, limit) =>
         })
         .slice(0, limit);
 
-const parseItems = (list, tryGet) =>
+const parseItems = (list, browser, tryGet) =>
     Promise.all(
         list.map((item) =>
             tryGet(item.link, async () => {
-                const { data: response } = await got(item.link, {
-                    cookieJar,
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await setCookies(page, cookie, 'cw.com.tw');
+                logger.debug(`Requesting ${item.link}`);
+                await page.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
                 });
 
+                const response = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
                 const $ = cheerio.load(response);
+
                 const meta = JSON.parse($('head script[type="application/ld+json"]').eq(0).text());
                 $('.article__head .breadcrumb, .article__head h1, .article__provideViews, .ad').remove();
                 $('img.lazyload').each((_, img) => {
@@ -67,9 +124,10 @@ const parseItems = (list, tryGet) =>
 
 module.exports = {
     baseUrl,
-    cookieJar,
-    got,
+    pathMap,
     getCookie,
+    setCookies,
+    parsePage,
     parseList,
     parseItems,
 };

package/lib/{routes → v2}/dlsite/campaign.js

@@ -139,31 +139,29 @@ module.exports = async (ctx) => {
     const title = `${info.name} | 割引中の作品`;
     const description = $('meta[name="description"]').attr('content');
     const list = $('tr[class]', '.n_worklist');
-    const item = list
-        .map((index, element) => {
-            const title = $('.work_name', element).text();
-            const link = $('.work_name > a', element).attr('href');
-            // 使链接
-            $('a', element).each((index, element) => {
-                $(element).attr('target', '_blank');
-            });
-            const description = $(element).html();
-            const arr = $('.search_tag', element);
-            const category = $('a', arr)
-                .map((index, a) => $(a).text())
-                .get();
-            const author = $('.maker_name', element).text();
+    const item = list.toArray().map((element) => {
+        const title = $('.work_name', element).text();
+        const link = $('.work_name > a', element).attr('href');
+        // 使链接
+        $('a', element).each((_index, element) => {
+            $(element).attr('target', '_blank');
+        });
+        const description = $(element).html();
+        const arr = $('.search_tag', element);
+        const category = $('a', arr)
+            .toArray()
+            .map((a) => $(a).text());
+        const author = $('.maker_name', element).text();
 
-            const signle = {
-                title,
-                link,
-                description,
-                category,
-                author,
-            };
-            return signle;
-        })
-        .get();
+        const signle = {
+            title,
+            link,
+            description,
+            category,
+            author,
+        };
+        return signle;
+    });
 
     ctx.state.data = {
         title,

package/lib/v2/dlsite/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/campaign/:type/:free?': ['cssxsh'],
+    '/new/:type': ['cssxsh'],
+};

package/lib/{routes → v2}/dlsite/new.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 const host = 'https://www.dlsite.com';
 const infos = {
@@ -69,35 +70,33 @@ module.exports = async (ctx) => {
     const list = $('.n_worklist_item');
     const dateText = $('.work_update')
         .text()
-        .replace(/(年 |月)/g, '-')
-        .substring(0, 10);
-    const pubDate = new Date(`${dateText} GMT+0900`).toUTCString();
-    const item = list
-        .map((index, element) => {
-            const title = $('.work_name', element).text();
-            const link = $('.work_name > a', element).attr('href');
-            // 使链接
-            $('a', element).each((index, element) => {
-                $(element).attr('target', '_blank');
-            });
-            const description = $(element).html();
-            const arr = $('.search_tag', element);
-            const category = $('a', arr)
-                .map((index, a) => $(a).text())
-                .get();
-            const author = $('.maker_name', element).text();
+        .trim()
+        .replace(/（.*）/g, '');
+    const pubDate = parseDate(dateText, 'YYYY年M月D日');
+    const item = list.toArray().map((element) => {
+        const title = $('.work_name', element).text();
+        const link = $('.work_name > a', element).attr('href');
+        // 使链接
+        $('a', element).each((_index, element) => {
+            $(element).attr('target', '_blank');
+        });
+        const description = $(element).html();
+        const arr = $('.search_tag', element);
+        const category = $('a', arr)
+            .toArray()
+            .map((a) => $(a).text());
+        const author = $('.maker_name', element).text();
 
-            const signle = {
-                title,
-                link,
-                description,
-                category,
-                author,
-                pubDate,
-            };
-            return signle;
-        })
-        .get();
+        const signle = {
+            title,
+            link,
+            description,
+            category,
+            author,
+            pubDate,
+        };
+        return signle;
+    });
 
     ctx.state.data = {
         title,

package/lib/v2/dlsite/radar.js

@@ -0,0 +1,15 @@
+module.exports = {
+    'dlsite.com': {
+        _name: 'DLsite',
+        '.': [
+            {
+                title: '当前日期发售的新产品',
+                docs: 'https://docs.rsshub.app/anime.html#dlsite',
+            },
+            {
+                title: '产品打折信息',
+                docs: 'https://docs.rsshub.app/anime.html#dlsite',
+            },
+        ],
+    },
+};

package/lib/v2/dlsite/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/campaign/:type/:free?', require('./campaign'));
+    router.get('/new/:type', require('./new'));
+};

package/lib/v2/dmzj/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:category?': ['vzz64'],
+};

package/lib/v2/dmzj/news.js

@@ -0,0 +1,26 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const url = `https://news.dmzj.com/${ctx.params.category || ''}`;
+    const $ = cheerio.load((await got(url)).data);
+    ctx.state.data = {
+        title: $('title').text(),
+        link: url,
+        item: $('.briefnews_con_li .li_img_de')
+            .map((_, item) => ({
+                title: $(item).find('h3 a').text(),
+                link: $(item).find('h3 a').attr('href'),
+                author: $(item).find('.head_con_p_o span:nth-child(3)').text().split('：')[1],
+                pubDate: timezone(parseDate($(item).find('.head_con_p_o span').first().text(), 'YYYY-MM-DD HH:mm'), +8),
+                description: $(item).find('p.com_about').text(),
+                category: $(item)
+                    .find('.u_comfoot a .bqwarp')
+                    .map((_, item) => $(item).text())
+                    .get(),
+            }))
+            .get(),
+    };
+};