rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/v2/creative-comic/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'creative-comic.tw': {
+        _name: 'CCC 創作集',
+        '.': [
+            {
+                title: '漫畫',
+                docs: 'https://docs.rsshub.app/anime.html#ccc-chuang-zuo-ji',
+                source: ['/book/:id/*'],
+                target: '/creative-comic/:id',
+            },
+        ],
+    },
+};

package/lib/v2/creative-comic/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/book/:id/:coverOnly?/:quality?', require('./book'));
+};

package/lib/v2/creative-comic/templates/chapter.art

@@ -0,0 +1,11 @@
+{{ if chapter }}
+    <p>{{ chapter.name }}</p>
+{{ /if }}
+{{ if pages }}
+    {{ each pages page }}
+        <br><img src="{{ page }}">
+    {{ /each }}
+{{ /if }}
+{{ if cover }}
+    <img src="{{ cover }}">
+{{ /if }}

package/lib/v2/creative-comic/utils.js

@@ -0,0 +1,96 @@
+const got = require('@/utils/got');
+const CryptoJS = require('crypto-js');
+
+const apiHost = 'https://api.creative-comic.tw';
+const siteHost = 'https://www.creative-comic.tw';
+const device = 'web_desktop';
+const DEFAULT_TOKEN = 'freeforccc2020reading';
+
+const getBook = (bookId, uuid) =>
+    got(`${apiHost}/book/${bookId}/info`, {
+        headers: {
+            device,
+            uuid,
+        },
+    });
+
+const getChapter = (id, uuid) =>
+    got(`${apiHost}/book/chapter/${id}`, {
+        headers: {
+            device,
+            uuid,
+        },
+    });
+
+const getChapters = (bookId, uuid) =>
+    got(`${apiHost}/book/${bookId}/chapter`, {
+        headers: {
+            device,
+            uuid,
+        },
+    });
+
+const getImgEncrypted = async (pageId, quality) => {
+    const { data: res } = await got(`${siteHost}/fs/chapter_content/encrypt/${pageId}/${quality}`, {
+        headers: {
+            device,
+        },
+        responseType: 'buffer',
+    });
+    return Buffer.from(res).toString('base64');
+};
+
+const getImgKey = (pageId, uuid) =>
+    got(`${apiHost}/book/chapter/image/${pageId}`, {
+        headers: {
+            device,
+            uuid,
+        },
+    });
+
+const getUuid = (tryGet) =>
+    tryGet('creative-comic:uuid', async () => {
+        const { data } = await got(`${apiHost}/guest`, {
+            headers: {
+                device,
+            },
+        });
+        return data.data;
+    });
+
+const decrypt = (encrypted, secrets) =>
+    CryptoJS.AES.decrypt(encrypted, CryptoJS.enc.Hex.parse(secrets.key), {
+        iv: CryptoJS.enc.Hex.parse(secrets.iv),
+        mode: CryptoJS.mode.CBC,
+        padding: CryptoJS.pad.Pkcs7,
+    }).toString(CryptoJS.enc.Utf8);
+
+const token2Key = (token) => {
+    const t = CryptoJS.SHA512(token).toString();
+    return {
+        key: t.substring(0, 64),
+        iv: t.substring(30, 62), // t.substr(30, 32)
+    };
+};
+
+const getRealKey = (imgKey, token = DEFAULT_TOKEN) => {
+    const secrets = token2Key(token);
+    const key = decrypt(imgKey, secrets);
+    const realKey = key.split(':');
+    return {
+        key: realKey[0],
+        iv: realKey[1],
+    };
+};
+
+module.exports = {
+    getBook,
+    getChapter,
+    getChapters,
+    getImgEncrypted,
+    getImgKey,
+    getUuid,
+    decrypt,
+    token2Key,
+    getRealKey,
+};

package/lib/v2/cts/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category': ['miles170'],
+};

package/lib/v2/cts/news.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const asyncPool = require('tiny-async-pool');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+    const currentUrl = `https://news.cts.com.tw/${category}/index.html`;
+    const response = await got(currentUrl);
+    const $ = cheerio.load(response.data);
+    const items = [];
+    for await (const data of asyncPool(5, $('#newslist-top a[title]').slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20), (item) => {
+        item = $(item);
+        const link = item.attr('href');
+        return ctx.cache.tryGet(link, async () => {
+            const response = await got(link);
+            const $ = cheerio.load(response.data);
+            const author = $('.artical-content p:eq(0)').text().trim();
+            $('.artical-content p:eq(0), .artical-content .flexbox').remove();
+
+            return {
+                title: item.attr('title'),
+                author,
+                description: $('.artical-content').html(),
+                category: $('meta[property="article:section"]').attr('content'),
+                pubDate: parseDate($('meta[property="article:published_time"]').attr('content')),
+                link,
+            };
+        });
+    })) {
+        items.push(data);
+    }
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        description: $('meta[name="description"]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/cts/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'cts.com.tw': {
+        _name: '華視',
+        news: [
+            {
+                title: '新聞',
+                docs: 'https://docs.rsshub.app/traditional-media.html#hua-shi-xin-wen',
+                source: '/:category/index.html',
+                target: '/cts/:category',
+            },
+        ],
+    },
+};

package/lib/v2/cts/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:category', require('./news'));
+};

package/lib/v2/cuc/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/yz': ['niuyi1017'],
+};

package/lib/v2/cuc/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'cuc.edu.cn': {
+        _name: '中国传媒大学',
+        yz: [
+            {
+                title: '研究生招生网',
+                docs: 'https://docs.rsshub.app/university.html#zhong-guo-chuan-mei-da-xue',
+                source: '/*',
+                target: '/',
+            },
+        ],
+    },
+};

package/lib/v2/cuc/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/yz', require('./yz'));
+};

package/lib/{routes/universities → v2}/cuc/yz.js

@@ -1,6 +1,5 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const url = require('url');
 const iconv = require('iconv-lite');
 
 const host = 'http://yz.cuc.edu.cn/';
@@ -18,13 +17,12 @@ module.exports = async (ctx) => {
     const $ = cheerio.load(responseHtml);
 
     const notice = $('#notice_area').children('.notice_block_top');
-    const content = notice.children('.notice_content1').children();
-
+    const content = notice.children('.notice_content1').children('p');
     const items = content
         .map((_, elem) => {
             const a = $('a', elem);
             return {
-                link: url.resolve(host, a.attr('href')),
+                link: new URL(a.attr('href'), host).href,
                 title: a.text(),
             };
         })

package/lib/v2/curius/links.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const username = ctx.params.name;
+
+    const name_response = await got(`https://curius.app/api/users/${username}`, {
+        headers: {
+            Referer: `https://curius.app/${username}`,
+        },
+    });
+
+    const data = name_response.data;
+
+    const uid = data.user.id;
+    const name = `${data.user.firstName} ${data.user.lastName}`;
+
+    const response = await got(`https://curius.app/api/users/${uid}/links?page=0`, {
+        headers: {
+            Referer: `https://curius.app/${username}`,
+        },
+    });
+
+    const items = response.data.userSaved.map((item) => ({
+        title: item.title,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            item,
+        }),
+        link: item.link,
+        pubDate: parseDate(item.createdDate),
+        guid: `curius:${username}:${item.id}`,
+    }));
+
+    ctx.state.data = {
+        title: `${name} - Curius`,
+        link: `https://curius.app/${username}`,
+        description: `${name} - Curius`,
+        allowEmpty: true,
+        item: items,
+    };
+};

package/lib/v2/curius/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/links/:name': ['Ovler-Young'],
+};

package/lib/v2/curius/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'curius.app': {
+        _name: 'Curius',
+        '.': [
+            {
+                title: '用户',
+                docs: 'https://docs.rsshub.app/social-media.html#curius',
+                source: '/:name',
+                target: '/curius/links/:name',
+            },
+        ],
+    },
+};

package/lib/v2/curius/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/links/:name', require('./links'));
+};

package/lib/v2/curius/templates/description.art

@@ -0,0 +1,16 @@
+{{ if item.metadata.full_text }}
+    原文：{{@ item.metadata.full_text.replace(/\n/gm, '<br>') }}<br><br>
+{{ /if }}
+{{ if item.comments.length }}
+    评论：{{ item.comments[0].text.substring(0, 100) }}<br><br>
+{{ /if }}
+{{ if item.highlights.length }}
+    评论：
+    {{ each item.highlights highlight }}
+        {{ if highlight.comment }}
+            {{ highlight.highlight }}<br>评论：{{ highlight.comment.text }}<br>
+        {{ else }}
+            <br>标注：{{ highlight.highlight }}<br>
+        {{ /if }}
+    {{ /each }}
+{{ /if }}

package/lib/v2/cw/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/master/:channel': ['TonyRL'],
+    '/sub/:channel': ['TonyRL'],
+    '/today': ['TonyRL'],
+};

package/lib/v2/cw/master.js

@@ -0,0 +1,31 @@
+const cheerio = require('cheerio');
+const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { channel } = ctx.params;
+    const pageUrl = `${baseUrl}/masterChannel.action`;
+    if (!cookieJar) {
+        await getCookie();
+    }
+    const { data: response } = await got(pageUrl, {
+        headers: {
+            Referer: baseUrl,
+        },
+        cookieJar,
+        searchParams: {
+            idMasterChannel: channel,
+        },
+    });
+    const $ = cheerio.load(response);
+
+    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
+    const items = await parseItems(list, ctx.cache.tryGet);
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('meta[name=description]').attr('content'),
+        image: `${baseUrl}/assets_new/img/fbshare.jpg`,
+        language: $('meta[property="og:locale"]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/cw/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'cw.com.tw': {
+        _name: '天下雜誌',
+        '.': [
+            {
+                title: '最新上線',
+                docs: 'https://docs.rsshub.app/traditional-media.html#tian-xia-za-zhi',
+                source: ['/today', '/'],
+                target: '/cw/today',
+            },
+            {
+                title: '主頻道',
+                docs: 'https://docs.rsshub.app/traditional-media.html#tian-xia-za-zhi',
+                source: ['/masterChannel.action'],
+                target: (_, url) => `/cw/master/${new URL(url).searchParams.get('idMasterChannel')}`,
+            },
+            {
+                title: '子頻道',
+                docs: 'https://docs.rsshub.app/traditional-media.html#tian-xia-za-zhi',
+                source: ['/subchannel.action'],
+                target: (_, url) => `/cw/sub/${new URL(url).searchParams.get('idSubChannel')}`,
+            },
+        ],
+    },
+};

package/lib/v2/cw/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/master/:channel', require('./master'));
+    router.get('/sub/:channel', require('./sub'));
+    router.get('/today', require('./today'));
+};

package/lib/v2/cw/sub.js

@@ -0,0 +1,31 @@
+const cheerio = require('cheerio');
+const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { channel } = ctx.params;
+    const pageUrl = `${baseUrl}/subchannel.action`;
+    if (!cookieJar) {
+        await getCookie();
+    }
+    const { data: response } = await got(pageUrl, {
+        headers: {
+            Referer: baseUrl,
+        },
+        cookieJar,
+        searchParams: {
+            idSubChannel: channel,
+        },
+    });
+    const $ = cheerio.load(response);
+
+    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
+    const items = await parseItems(list, ctx.cache.tryGet);
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('meta[name=description]').attr('content'),
+        image: `${baseUrl}/assets_new/img/fbshare.jpg`,
+        language: $('meta[property="og:locale"]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/cw/today.js

@@ -0,0 +1,27 @@
+const cheerio = require('cheerio');
+const { baseUrl, cookieJar, got, parseList, parseItems, getCookie } = require('./utils');
+
+module.exports = async (ctx) => {
+    const pageUrl = `${baseUrl}/today`;
+    if (!cookieJar) {
+        await getCookie();
+    }
+    const { data: response } = await got(pageUrl, {
+        headers: {
+            Referer: baseUrl,
+        },
+        cookieJar,
+    });
+    const $ = cheerio.load(response);
+
+    const list = parseList($, ctx.query.limit ? Number(ctx.query.limit) : 100);
+    const items = await parseItems(list, ctx.cache.tryGet);
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('meta[name=description]').attr('content'),
+        image: `${baseUrl}/assets_new/img/fbshare.jpg`,
+        language: $('meta[property="og:locale"]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/cw/utils.js

@@ -0,0 +1,75 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { Cookie, CookieJar } = require('tough-cookie');
+let cookieJar;
+const config = require('@/config').value;
+
+const baseUrl = 'https://www.cw.com.tw';
+
+const got = require('@/utils/got').extend({
+    headers: {
+        'User-Agent': config.trueUA,
+    },
+});
+
+const getCookie = async () => {
+    const response = await got(`${baseUrl}/user/get/cookie-bar`);
+    const cookies = response.headers['set-cookie'];
+    if (Array.isArray(cookies)) {
+        cookieJar = cookies.map(Cookie.parse);
+    } else {
+        cookieJar = [Cookie.parse(cookieJar)];
+    }
+    cookieJar = CookieJar.fromJSON({ cookies: cookieJar });
+};
+
+const parseList = ($, limit) =>
+    $('.caption')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('h3').text(),
+                link: item.find('h3 a').attr('href'),
+                pubDate: parseDate(item.find('time').text()),
+            };
+        })
+        .slice(0, limit);
+
+const parseItems = (list, tryGet) =>
+    Promise.all(
+        list.map((item) =>
+            tryGet(item.link, async () => {
+                const { data: response } = await got(item.link, {
+                    cookieJar,
+                });
+
+                const $ = cheerio.load(response);
+                const meta = JSON.parse($('head script[type="application/ld+json"]').eq(0).text());
+                $('.article__head .breadcrumb, .article__head h1, .article__provideViews, .ad').remove();
+                $('img.lazyload').each((_, img) => {
+                    if (img.attribs['data-src']) {
+                        img.attribs.src = img.attribs['data-src'];
+                        delete img.attribs['data-src'];
+                    }
+                });
+
+                item.title = $('head title').text();
+                item.category = $('meta[name=keywords]').attr('content').split(',');
+                item.pubDate = parseDate(meta.datePublished);
+                item.author = meta.author.name.replace(',', ' ') || meta.publisher.name;
+                item.description = $('.article__head .container').html() + $('.article__content').html();
+
+                return item;
+            })
+        )
+    );
+
+module.exports = {
+    baseUrl,
+    cookieJar,
+    got,
+    getCookie,
+    parseList,
+    parseItems,
+};

package/lib/v2/dcard/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/posts/:type?': ['DIYgod'], // alias
+    '/:section/:type?': ['HenryQW'],
+};

package/lib/v2/dcard/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'dcard.tw': {
+        _name: 'Dcard',
+        www: [
+            {
+                title: '首頁帖子-最新',
+                docs: 'https://docs.rsshub.app/bbs.html#dcard',
+                source: '/f',
+                target: '/dcard/posts/latest',
+            },
+            {
+                title: '首頁帖子-熱門',
+                docs: 'https://docs.rsshub.app/bbs.html#dcard',
+                source: '/f',
+                target: '/dcard/posts/popular',
+            },
+            {
+                title: '板塊帖子-最新',
+                docs: 'https://docs.rsshub.app/bbs.html#dcard',
+                source: '/f/:section',
+                target: '/dcard/:section/latest',
+            },
+            {
+                title: '板塊帖子-熱門',
+                docs: 'https://docs.rsshub.app/bbs.html#dcard',
+                source: '/f/:section',
+                target: '/dcard/:section/popular',
+            },
+        ],
+    },
+};

package/lib/v2/dcard/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:section/:type?', require('./section'));
+};

package/lib/v2/dcard/section.js

@@ -0,0 +1,66 @@
+const { parseDate } = require('@/utils/parse-date');
+const utils = require('./utils');
+
+module.exports = async (ctx) => {
+    const { type = 'latest', section = 'posts' } = ctx.params;
+    const limit = ctx.query.limit ? Number(ctx.query.limit) : 30;
+    const browser = await require('@/utils/puppeteer')();
+
+    let link = `https://www.dcard.tw/f`;
+    let api = `https://www.dcard.tw/service/api/v2`;
+    let title = `Dcard - `;
+
+    if (section !== 'posts' && section !== 'popular' && section !== 'latest') {
+        link += `/${section}`;
+        api += `/forums/${section}`;
+        title += `${section} - `;
+    }
+    api += `/posts`;
+    if (type === 'popular') {
+        link += '?latest=false';
+        api += '?popular=true';
+        title += '熱門';
+    } else {
+        link += '?latest=true';
+        api += '?popular=false';
+        title += '最新';
+    }
+
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.setExtraHTTPHeaders({
+        referer: `https://www.dcard.tw/f/${section}`,
+    });
+
+    await page.goto(`${api}&limit=100`);
+    await page.waitForSelector('body > pre');
+    const response = await page.evaluate(() => document.querySelector('body > pre').innerText);
+    const cookies = await ctx.cache.tryGet('dcard:cookies', async () => await page.cookies(), 3600, false);
+    await page.close();
+
+    const data = JSON.parse(response);
+    const items = data.map((item) => ({
+        title: `「${item.forumName}」${item.title}`,
+        link: `https://www.dcard.tw/f/${item.forumAlias}/p/${item.id}`,
+        description: item.excerpt,
+        author: `${item.school || '匿名'}．${item.gender === 'M' ? '男' : '女'}`,
+        pubDate: parseDate(item.createdAt),
+        category: [item.forumName, ...item.topics],
+        forumAlias: item.forumAlias,
+        id: item.id,
+    }));
+
+    // parse fulltext for first `limit` items
+    const result = await utils.ProcessFeed(items, cookies, browser, limit, ctx.cache);
+    await browser.close();
+
+    ctx.state.data = {
+        title,
+        link,
+        description: '不想錯過任何有趣的話題嗎？趕快加入我們吧！',
+        item: result,
+    };
+};