rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f796ff1

package/lib/v2/cw/utils.js

@@ -0,0 +1,75 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { Cookie, CookieJar } = require('tough-cookie');
+let cookieJar;
+const config = require('@/config').value;
+
+const baseUrl = 'https://www.cw.com.tw';
+
+const got = require('@/utils/got').extend({
+    headers: {
+        'User-Agent': config.trueUA,
+    },
+});
+
+const getCookie = async () => {
+    const response = await got(`${baseUrl}/user/get/cookie-bar`);
+    const cookies = response.headers['set-cookie'];
+    if (Array.isArray(cookies)) {
+        cookieJar = cookies.map(Cookie.parse);
+    } else {
+        cookieJar = [Cookie.parse(cookieJar)];
+    }
+    cookieJar = CookieJar.fromJSON({ cookies: cookieJar });
+};
+
+const parseList = ($, limit) =>
+    $('.caption')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('h3').text(),
+                link: item.find('h3 a').attr('href'),
+                pubDate: parseDate(item.find('time').text()),
+            };
+        })
+        .slice(0, limit);
+
+const parseItems = (list, tryGet) =>
+    Promise.all(
+        list.map((item) =>
+            tryGet(item.link, async () => {
+                const { data: response } = await got(item.link, {
+                    cookieJar,
+                });
+
+                const $ = cheerio.load(response);
+                const meta = JSON.parse($('head script[type="application/ld+json"]').eq(0).text());
+                $('.article__head .breadcrumb, .article__head h1, .article__provideViews, .ad').remove();
+                $('img.lazyload').each((_, img) => {
+                    if (img.attribs['data-src']) {
+                        img.attribs.src = img.attribs['data-src'];
+                        delete img.attribs['data-src'];
+                    }
+                });
+
+                item.title = $('head title').text();
+                item.category = $('meta[name=keywords]').attr('content').split(',');
+                item.pubDate = parseDate(meta.datePublished);
+                item.author = meta.author.name.replace(',', ' ') || meta.publisher.name;
+                item.description = $('.article__head .container').html() + $('.article__content').html();
+
+                return item;
+            })
+        )
+    );
+
+module.exports = {
+    baseUrl,
+    cookieJar,
+    got,
+    getCookie,
+    parseList,
+    parseItems,
+};

package/lib/v2/dcard/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/posts/:type?': ['DIYgod'], // alias
+    '/:section/:type?': ['HenryQW'],
+};

package/lib/v2/dcard/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'dcard.tw': {
+        _name: 'Dcard',
+        www: [
+            {
+                title: '首頁帖子-最新',
+                docs: 'https://docs.rsshub.app/bbs.html#dcard',
+                source: '/f',
+                target: '/dcard/posts/latest',
+            },
+            {
+                title: '首頁帖子-熱門',
+                docs: 'https://docs.rsshub.app/bbs.html#dcard',
+                source: '/f',
+                target: '/dcard/posts/popular',
+            },
+            {
+                title: '板塊帖子-最新',
+                docs: 'https://docs.rsshub.app/bbs.html#dcard',
+                source: '/f/:section',
+                target: '/dcard/:section/latest',
+            },
+            {
+                title: '板塊帖子-熱門',
+                docs: 'https://docs.rsshub.app/bbs.html#dcard',
+                source: '/f/:section',
+                target: '/dcard/:section/popular',
+            },
+        ],
+    },
+};

package/lib/v2/dcard/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:section/:type?', require('./section'));
+};

package/lib/v2/dcard/section.js

@@ -0,0 +1,66 @@
+const { parseDate } = require('@/utils/parse-date');
+const utils = require('./utils');
+
+module.exports = async (ctx) => {
+    const { type = 'latest', section = 'posts' } = ctx.params;
+    const limit = ctx.query.limit ? Number(ctx.query.limit) : 30;
+    const browser = await require('@/utils/puppeteer')();
+
+    let link = `https://www.dcard.tw/f`;
+    let api = `https://www.dcard.tw/service/api/v2`;
+    let title = `Dcard - `;
+
+    if (section !== 'posts' && section !== 'popular' && section !== 'latest') {
+        link += `/${section}`;
+        api += `/forums/${section}`;
+        title += `${section} - `;
+    }
+    api += `/posts`;
+    if (type === 'popular') {
+        link += '?latest=false';
+        api += '?popular=true';
+        title += '熱門';
+    } else {
+        link += '?latest=true';
+        api += '?popular=false';
+        title += '最新';
+    }
+
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.setExtraHTTPHeaders({
+        referer: `https://www.dcard.tw/f/${section}`,
+    });
+
+    await page.goto(`${api}&limit=100`);
+    await page.waitForSelector('body > pre');
+    const response = await page.evaluate(() => document.querySelector('body > pre').innerText);
+    const cookies = await ctx.cache.tryGet('dcard:cookies', async () => await page.cookies(), 3600, false);
+    await page.close();
+
+    const data = JSON.parse(response);
+    const items = data.map((item) => ({
+        title: `「${item.forumName}」${item.title}`,
+        link: `https://www.dcard.tw/f/${item.forumAlias}/p/${item.id}`,
+        description: item.excerpt,
+        author: `${item.school || '匿名'}．${item.gender === 'M' ? '男' : '女'}`,
+        pubDate: parseDate(item.createdAt),
+        category: [item.forumName, ...item.topics],
+        forumAlias: item.forumAlias,
+        id: item.id,
+    }));
+
+    // parse fulltext for first `limit` items
+    const result = await utils.ProcessFeed(items, cookies, browser, limit, ctx.cache);
+    await browser.close();
+
+    ctx.state.data = {
+        title,
+        link,
+        description: '不想錯過任何有趣的話題嗎？趕快加入我們吧！',
+        item: result,
+    };
+};

package/lib/v2/dcard/utils.js

@@ -0,0 +1,50 @@
+const asyncPool = require('tiny-async-pool');
+
+const ProcessFeed = async (items, cookies, browser, limit, cache) => {
+    let newCookies = [];
+    const result = [];
+    for await (const item of asyncPool(3, items.slice(0, limit), async (i) => {
+        const url = `https://www.dcard.tw/service/api/v2/posts/${i.id}`;
+        const content = await cache.tryGet(`dcard:${i.id}`, async () => {
+            let response;
+            // try catch 处理被删除的帖子
+            try {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'fetch' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+                });
+                await page.setExtraHTTPHeaders({
+                    referer: `https://www.dcard.tw/f/${i.forumAlias}/p/${i.id}`,
+                });
+                await page.setCookie(...cookies);
+                await page.goto(url);
+                await page.waitForSelector('body > pre');
+                response = await page.evaluate(() => document.querySelector('body > pre').innerText);
+                newCookies = await page.cookies();
+                await page.close();
+
+                const data = JSON.parse(response);
+                let body = data.content;
+                body = body.replace(/(?=https?:\/\/).*?(?<=\.(jpe?g|gif|png))/gi, (m) => `<img src="${m}">`);
+                body = body.replace(/(?=https?:\/\/).*(?<!jpe?g"?>?)$/gim, (m) => `<a href="${m}">${m}</a>`);
+                body = body.replace(/\n/g, '<br>');
+
+                return body;
+            } catch (error) {
+                return '';
+            }
+        });
+
+        i.description = content;
+        return i;
+    })) {
+        result.push(item);
+    }
+    await cache.set('dcard:cookies', newCookies, 3600);
+    return [...result, ...items.slice(limit)];
+};
+
+module.exports = {
+    ProcessFeed,
+};

package/lib/{routes → v2}/dedao/index.js

@@ -1,9 +1,11 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const caty = ctx.params.caty || 'news';
-    const rootUrl = `https://www.igetget.com/${caty === 'video' ? 'video' : 'news'}`;
+    const category = ctx.params.category ?? 'news';
+
+    const rootUrl = `https://www.igetget.com/${category === 'video' ? 'video' : 'news'}`;
 
     const response = await got({
         method: 'get',
@@ -12,19 +14,20 @@ module.exports = async (ctx) => {
 
     const data = JSON.parse(response.data.match(/window.__INITIAL_STATE__= (.*);<\/script>/)[1]);
 
-    const list = (caty === 'news' ? data.news : caty === 'figure' ? data.figure : data.videoList).map((item) => ({
+    let items = (category === 'news' ? data.news : category === 'figure' ? data.figure : data.videoList).map((item) => ({
         title: item.title,
-        pubDate: new Date(item.online_time).toUTCString(),
-        link: `${rootUrl}/${caty === 'news' ? 'article/' : caty === 'figure' ? 'people/' : ''}${item.online_time.split('T')[0].split('-').join('')}/${item.token}`,
+        pubDate: parseDate(item.online_time),
+        link: `${rootUrl}/${category === 'news' ? 'article/' : category === 'figure' ? 'people/' : ''}${item.online_time.split('T')[0].split('-').join('')}/${item.token}`,
     }));
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',
                     url: item.link,
                 });
+
                 const content = cheerio.load(detailResponse.data);
 
                 item.description = content('.menu-article').html();
@@ -35,7 +38,7 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: `得到${caty === 'video' ? '' : '大事件'} - ${caty === 'news' ? '新闻' : caty === 'figure' ? '人物故事' : '视频'}`,
+        title: `得到${category === 'video' ? '' : '大事件'} - ${category === 'news' ? '新闻' : category === 'figure' ? '人物故事' : '视频'}`,
         link: rootUrl,
         item: items,
         description: data.description,

package/lib/v2/dedao/knowledge.js

@@ -0,0 +1,64 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const topic = ctx.params.topic ?? '';
+    const type = /t|y/i.test(ctx.params.type ?? 'true');
+    const count = ctx.query.limit ? parseInt(ctx.query.limit) : 100;
+
+    const rootUrl = 'https://www.dedao.cn';
+    const currentUrl = `${rootUrl}/knowledge${topic === '' ? '' : '/topic/' + topic}`;
+    const apiUrl = `${rootUrl}/pc/ledgers/${topic === '' ? 'notes/friends_timeline' : 'topic/notes/list'}`;
+    const detailUrl = `${rootUrl}/pc/ledgers/topic/detail`;
+
+    let title = '',
+        description = '';
+
+    if (topic !== '') {
+        const detailResponse = await got({
+            method: 'post',
+            url: detailUrl,
+            json: {
+                incr_view_count: false,
+                topic_id_hazy: topic,
+            },
+        });
+
+        title = detailResponse.data.c.name;
+        description = detailResponse.data.c.intro;
+    }
+
+    const response = await got({
+        method: 'post',
+        url: apiUrl,
+        json: {
+            count,
+            load_chain: true,
+            is_elected: type,
+            page_id: 0,
+            topic_id_hazy: topic,
+            version: 2,
+        },
+    });
+
+    const items = (topic === '' ? response.data.c.notes : response.data.c.note_detail_list).map((item) => ({
+        title: item.f_part.note,
+        author: item.f_part.nick_name,
+        link: `${rootUrl}/knowledge/note/${item.f_part.note_id_hazy}`,
+        pubDate: parseDate(item.f_part.time_desc, 'MM-DD'),
+        description: art(path.join(__dirname, 'templates/knowledge.art'), {
+            rootUrl,
+            f_part: item.f_part,
+            s_part: item.s_part,
+        }),
+    }));
+
+    ctx.state.data = {
+        title: `得到 - 知识城邦${title === '' ? '' : ' - ' + title}`,
+        link: currentUrl,
+        item: items,
+        description,
+    };
+};

package/lib/{routes → v2}/dedao/list.js

@@ -2,20 +2,24 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 
 module.exports = async (ctx) => {
-    const caty = ctx.params.caty || '年度日更';
+    const category = ctx.params.category ?? '年度日更';
+
     const rootUrl = 'https://www.igetget.com';
+
     const response = await got({
         method: 'get',
         url: rootUrl,
     });
 
     const listUrl = `${rootUrl}${response.data.match(new RegExp('<a href="(.*)">年度日更<\\/a>'))[1]}`;
+
     const listResponse = await got({
         method: 'get',
         url: listUrl,
     });
 
-    const currentUrl = `${rootUrl}${listResponse.data.match(new RegExp('<span>' + caty + '<\\/span><a href="(.*)" rel="tag"><\\/a>'))[1].split('"')[0]}`;
+    const currentUrl = `${rootUrl}${listResponse.data.match(new RegExp('<span>' + category + '<\\/span><a href="(.*)" rel="tag"><\\/a>'))[1].split('"')[0]}`;
+
     const currentResponse = await got({
         method: 'get',
         url: currentUrl,
@@ -23,24 +27,26 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(currentResponse.data);
 
-    const list = $('.pro-info p a')
-        .slice(0, 10)
-        .map((_, item) => {
+    let items = $('.pro-info p a')
+        .toArray()
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 10)
+        .map((item) => {
             item = $(item);
+
             return {
                 title: item.text(),
                 link: `${rootUrl}${item.attr('href')}`,
             };
-        })
-        .get();
+        });
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',
                     url: item.link,
                 });
+
                 const content = cheerio.load(detailResponse.data);
 
                 content('.more-bt').remove();
@@ -53,7 +59,7 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: `得到 - ${caty}`,
+        title: `得到 - ${category}`,
         link: currentUrl,
         item: items,
     };

package/lib/v2/dedao/maintainer.js

@@ -0,0 +1,8 @@
+module.exports = {
+    '/figure': ['nczitzk'],
+    '/knowledge/:topic?/:type?': ['nczitzk'],
+    '/list/:category?': ['nczitzk'],
+    '/news': ['nczitzk'],
+    '/user/:id/:type?': ['nczitzk'],
+    '/video': ['nczitzk'],
+};

package/lib/v2/dedao/radar.js

@@ -0,0 +1,50 @@
+module.exports = {
+    'dedao.cn': {
+        _name: '得到',
+        '.': [
+            {
+                title: '知识城邦',
+                docs: 'https://docs.rsshub.app/new-media.html#de-dao-zhi-shi-cheng-bang',
+                source: ['/knowledge/topic/:topic', '/knowledge', '/'],
+                target: '/dedao/knowledge/:topic?/:type?',
+            },
+        ],
+    },
+    'igetget.com': {
+        _name: '得到',
+        '.': [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/new-media.html#de-dao-shou-ye',
+                source: ['/'],
+                target: '/dedao/list/:category?',
+            },
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#de-dao-xin-wen',
+                source: ['/news', '/'],
+                target: '/dedao/news',
+            },
+            {
+                title: '人物故事',
+                docs: 'https://docs.rsshub.app/new-media.html#de-dao-ren-wu-gu-shi',
+                source: ['/news', '/'],
+                target: '/dedao/figure',
+            },
+            {
+                title: '视频',
+                docs: 'https://docs.rsshub.app/new-media.html#de-dao-shi-pin',
+                source: ['/video', '/'],
+                target: '/dedao/video',
+            },
+        ],
+        m: [
+            {
+                title: '用户主页',
+                docs: 'https://docs.rsshub.app/new-media.html#de-dao-yong-hu-zhu-ye',
+                source: ['/native/mine/account', '/'],
+                target: (params, url) => `/dedao/user/${new URL(url).searchParams.get('enId')}`,
+            },
+        ],
+    },
+};

package/lib/v2/dedao/router.js

@@ -0,0 +1,6 @@
+module.exports = function (router) {
+    router.get('/knowledge/:topic?/:type?', require('./knowledge'));
+    router.get('/list/:category?', require('./list'));
+    router.get('/user/:id/:type?', require('./user'));
+    router.get('/:category?', require('./index'));
+};

package/lib/v2/dedao/templates/knowledge.art

@@ -0,0 +1,29 @@
+{{ if f_part.note }}
+<p>{{@ f_part.note
+    .replace(/\n\n/g, '</p><p>')
+    .replace(/<≧❆>{"name":"(.*?)","uid":"\d+","at":"1"}<\/≦❆>/g, ' @$1')
+    }}</p>
+{{ /if }}
+{{ if f_part.images }}
+<br>
+{{ set f_part_images = f_part.images }}
+{{ each f_part_images image }}
+<img src="{{ image.match(/"url":"(.*?)"/)[1] }}">
+{{ /each }}
+{{ /if }}
+{{ if s_part }}
+<br>
+<p>引用 {{ s_part.nick_name }}{{ if s_part.v_info }} ({{ s_part.v_info }}){{ /if }}:</p>
+<p>{{@ s_part.note
+    .replace(/\n\n/g, '</p><p>')
+    .replace(/<≧❆>{"name":"(.*?)","uid":"\d+","at":"1"}<\/≦❆>/g, ' @$1')
+    }}</p>
+<a href="{{ rootUrl }}/knowledge/note/{{ s_part.note_id_hazy }}">[查看原文]</a>
+{{ if s_part.images }}
+<br>
+{{ set s_part_images = s_part.images }}
+{{ each s_part_images image}}
+<img src="{{ image.match(/"url":"(.*?)"/)[1] }}">
+{{ /each }}
+{{ /if }}
+{{ /if }}

package/lib/v2/dedao/templates/user.art

@@ -0,0 +1,38 @@
+{{ if name }}
+<p>{{@ content
+    .replace(/\n\n/g, '</p><p>')
+    .replace(/<≧❆>{"name":"(.*?)","uid":"\d+","at":"1"}<\/≦❆>/g, ' @$1')
+    }}</p>
+<br>
+<p>引用 {{ name }}{{ if vinfo }} ({{ vinfo }}){{ /if }}:</p>
+{{ /if }}
+<p>{{@ note
+    .replace(/\n\n/g, '</p><p>')
+    .replace(/<≧❆>{"name":"(.*?)","uid":"\d+","at":"1"}<\/≦❆>/g, ' @$1')
+    }}</p>
+{{ if extra }}
+<br>
+{{ if extra.img }}
+{{ if extra.share_ext }}
+<a href="{{ extra.share_ext.h_5_url }}">
+{{ /if }}
+<figure>
+    <img src="{{ extra.img }}">
+    <figcaption>{{ extra.title }}</figcaption>
+</figure>
+{{ if extra.share_ext }}
+</a>
+{{ /if }}
+{{ /if }}
+{{ if extra.images }}
+{{ set extra_images = extra.images }}
+{{ each extra_images image }}
+<img src="{{ image.match(/"url":"(.*?)"/)[1] }}">
+{{ /each }}
+{{ /if }}
+{{ if video }}
+<figure>
+    <img src="{{ video.match(/"url":"(.*?)"/)[1] }}">
+</figure>
+{{ /if }}
+{{ /if }}

package/lib/v2/dedao/user.js

@@ -0,0 +1,79 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const types = {
+    0: '动态',
+    7: '书评',
+    12: '视频',
+};
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? '';
+    const type = ctx.params.type ? parseInt(ctx.params.type) : 0;
+    const count = ctx.query.limit ? parseInt(ctx.query.limit) : 100;
+
+    const rootUrl = 'https://m.igetget.com';
+    const currentUrl = `${rootUrl}/native/mine/account#/user/detail?enId=${id}`;
+    const apiUrl = `${rootUrl}/native/api/homePage/topicNote`;
+    const infoUrl = `${rootUrl}/native/api/homePage/userInfo`;
+
+    const detailResponse = await got({
+        method: 'post',
+        url: infoUrl,
+        json: {
+            hazy: id,
+        },
+    });
+
+    const data = detailResponse.data;
+
+    const author = data.c.nickname;
+    const image = data.c.avatar;
+    const description = `${data.c.v_info}: ${data.c.slogan}`;
+
+    const response = await got({
+        method: 'post',
+        url: apiUrl,
+        json: {
+            uid: null,
+            max_id_str: '0',
+            count,
+            max_createtime: 0,
+            is_only_repost: 0,
+            load_chain: true,
+            load_tag: 1,
+            source: 0,
+            note_type: type,
+            only_origin: false,
+            with_highlight: true,
+            hazy: id,
+        },
+    });
+
+    const items = response.data.c.list.map((item) => ({
+        author,
+        title: item.content || item.note || item.extra.title,
+        link: item.share_url,
+        pubDate: parseDate(item.create_time * 1000),
+        description: art(path.join(__dirname, 'templates/user.art'), {
+            rootUrl,
+            name: item.origin_notes_owner.name,
+            vinfo: item.origin_notes_owner.Vinfo,
+            content: item.content,
+            note: item.note,
+            extra: item.extra,
+            video: item.video.video_cover,
+        }),
+    }));
+
+    ctx.state.data = {
+        title: `${author}的得到主页 - ${types[type]}`,
+        link: currentUrl,
+        item: items,
+        image,
+        description,
+        allowEmpty: true,
+    };
+};

package/lib/v2/deepmind/blog.js

@@ -0,0 +1,32 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const parser = require('@/utils/rss-parser');
+
+module.exports = async (ctx) => {
+    const feed = await parser.parseURL('https://www.deepmind.com/blog/rss.xml');
+
+    const items = await Promise.all(
+        feed.items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+
+                item.description = $('.e_container .c_rich-text__cms').html();
+
+                delete item.content;
+                delete item.contentSnippet;
+                delete item.isoDate;
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: feed.title,
+        description: feed.description,
+        image: 'https://assets-global.website-files.com/621d30e84caf0be3291dbf1c/621d336835a91420c6a8dcf2_webclip.png',
+        link: `${feed.link}/blog`,
+        item: items,
+        language: 'en',
+    };
+};

package/lib/v2/deepmind/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/blog': ['nczitzk', 'TonyRL'],
+};