rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/lib/v2/bloomberg/radar.js

@@ -1,13 +1,19 @@
 module.exports = {
     'bloomberg.com': {
-        _name: 'bloomberg',
+        _name: 'Bloomberg',
         www: [
             {
-                title: 'Bloomberg',
-                docs: 'https://docs.rsshub.app/finance.html##bloomberg-news',
+                title: 'News',
+                docs: 'https://docs.rsshub.app/finance.html#bloomberg-news',
                 source: ['/:site', '/'],
                 target: '/bloomberg/:site?',
             },
+            {
+                title: 'Authors',
+                docs: 'https://docs.rsshub.app/finance.html#bloomberg',
+                source: ['/*/authors/:id/:slug', '/authors/:id/:slug'],
+                target: '/bloomberg/authors/:id/:slug',
+            },
         ],
     },
 };

package/lib/v2/bloomberg/router.js

@@ -1,4 +1,5 @@
 module.exports = function (router) {
+    router.get('/authors/:id/:slug/:source?', require('./authors'));
     router.get('/:site', require('./index'));
     router.get('/', require('./index'));
 };

package/lib/v2/bloomberg/utils.js

@@ -1,5 +1,6 @@
 const cheerio = require('cheerio');
 const path = require('path');
+const asyncPool = require('tiny-async-pool');
 
 const { parseDate } = require('@/utils/parse-date');
 const got = require('@/utils/got');
@@ -22,7 +23,7 @@ const apiEndpoints = {
     },
     videos: {
         url: 'https://www.bloomberg.com/news/videos/',
-        sel: '.footer-container + script',
+        sel: 'script',
     },
     newsletters: {
         url: 'https://www.bloomberg.com/news/newsletters/',
@@ -152,7 +153,9 @@ const parseAudioPage = async (res, api, item) => {
 };
 
 const parseVideoPage = async (res, api, item) => {
-    const json = cheerio.load(res.data)(api.sel).text().trim().replace('window.__PRELOADED_STATE__ = ', '').slice(0, -1);
+    const $ = cheerio.load(res.data);
+    const script = $(api.sel).filter((i, el) => $(el).text().includes('__PRELOADED_STATE__'));
+    const json = script.text().trim().replace('window.__PRELOADED_STATE__ = ', '').slice(0, -1);
     const article_json = JSON.parse(json);
     const video_story = article_json.video?.videoStory ?? article_json.quicktakeVideo?.videoStory;
     if (video_story) {
@@ -388,8 +391,17 @@ const processBody = async (body_html, story_json) => {
     return $.html();
 };
 
+const asyncPoolAll = async (...args) => {
+    const results = [];
+    for await (const result of asyncPool(...args)) {
+        results.push(result);
+    }
+    return results;
+};
+
 module.exports = {
     rootUrl,
+    asyncPoolAll,
     parseNewsList,
     parseArticle,
 };

package/lib/v2/btzj/index.js

@@ -4,10 +4,15 @@ const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const config = require('@/config').value;
+const allowDomain = ['2btjia.com', '88btbtt.com', 'btbtt15.com', 'btbtt20.com'];
 
 module.exports = async (ctx) => {
     let category = ctx.params.category ?? '';
-    let domain = ctx.query.domain ?? 'btbtt20.com';
+    let domain = ctx.query.domain ?? 'btbtt15.com';
+    if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(new URL(`http://${domain}/`).hostname)) {
+        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+    }
 
     if (category === 'base') {
         category = '';

package/lib/v2/byteclicks/index.js

@@ -0,0 +1,22 @@
+const got = require('@/utils/got');
+const { parseItem } = require('./utils');
+const baseUrl = 'https://byteclicks.com';
+
+module.exports = async (ctx) => {
+    const { data } = await got(`${baseUrl}/wp-json/wp/v2/posts`, {
+        searchParams: {
+            per_page: ctx.query.limit ? parseInt(ctx.query.limit) : 100,
+        },
+    });
+
+    const items = parseItem(data);
+
+    ctx.state.data = {
+        title: '字节点击 - 聚合全球优质资源，跟踪世界前沿科技',
+        description:
+            'byteclicks.com 最专业的前沿科技网站。聚合全球优质资源，跟踪世界前沿科技，精选推荐一些很棒的互联网好资源好工具好产品。寻找有前景好项目、找论文、找报告、找数据、找课程、找电子书上byteclicks！byteclicks.com是投资人、科研学者、学生每天必看的网站。',
+        image: 'https://byteclicks.com/wp-content/themes/RK-Blogger/images/wbolt.ico',
+        link: baseUrl,
+        item: items,
+    };
+};

package/lib/v2/byteclicks/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/': ['TonyRL'],
+    '/tag/:tag': ['TonyRL'],
+};

package/lib/v2/byteclicks/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'byteclicks.com': {
+        _name: '字节点击',
+        '.': [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/new-media.html#zi-jie-dian-ji',
+                source: ['/'],
+                target: '/byteclicks',
+            },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/new-media.html#zi-jie-dian-ji',
+                source: ['/tag/:tag'],
+                target: '/byteclicks/tag/:tag',
+            },
+        ],
+    },
+};

package/lib/v2/byteclicks/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/', require('./index'));
+    router.get('/tag/:tag', require('./tag'));
+};

package/lib/v2/byteclicks/tag.js

@@ -0,0 +1,30 @@
+const got = require('@/utils/got');
+const { parseItem } = require('./utils');
+const baseUrl = 'https://byteclicks.com';
+
+module.exports = async (ctx) => {
+    const { tag } = ctx.params;
+    const { data: search } = await got(`${baseUrl}/wp-json/wp/v2/tags`, {
+        searchParams: {
+            search: tag,
+            per_page: 100,
+        },
+    });
+    const tagData = search.find((item) => item.name === tag);
+
+    const { data } = await got(`${baseUrl}/wp-json/wp/v2/posts`, {
+        searchParams: {
+            per_page: ctx.query.limit ? parseInt(ctx.query.limit) : 100,
+            tags: tagData.id,
+        },
+    });
+
+    const items = parseItem(data);
+
+    ctx.state.data = {
+        title: `${tagData.name} - 字节点击`,
+        image: 'https://byteclicks.com/wp-content/themes/RK-Blogger/images/wbolt.ico',
+        link: tagData.link,
+        item: items,
+    };
+};

package/lib/v2/byteclicks/utils.js

@@ -0,0 +1,13 @@
+const { parseDate } = require('@/utils/parse-date');
+
+const parseItem = (data) =>
+    data.map((item) => ({
+        title: item.title.rendered,
+        description: item.content.rendered,
+        pubDate: parseDate(item.date_gmt),
+        link: item.link,
+    }));
+
+module.exports = {
+    parseItem,
+};

package/lib/v2/caixin/article.js

@@ -0,0 +1,27 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { data: response } = await got('https://mapiv5.caixin.com/m/api/getWapIndexListByPage');
+
+    const list = response.data.list.map((item) => ({
+        title: item.title,
+        description: item.summary,
+        author: item.author_name,
+        pubDate: parseDate(item.time, 'X'),
+        link: item.web_url,
+        pics: item.pics,
+        audio: item.cms_audio_url,
+        audio_image_url: item.audio_image_url,
+    }));
+
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: '财新网 - 首页',
+        link: 'https://www.caixin.com',
+        description: '财新网 - 首页',
+        item: items,
+    };
+};

package/lib/v2/caixin/blog.js

@@ -0,0 +1,58 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
+const { parseDate } = require('@/utils/parse-date');
+const { parseBlogArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { column } = ctx.params;
+    if (!isValidHost(column)) {
+        throw Error('Invalid column');
+    }
+    const link = `https://${column}.blog.caixin.com`;
+    const { data: response } = await got(link);
+    const $ = cheerio.load(response);
+    const authorId = $('script[type="text/javascript"]')
+        .text()
+        .match(/window\.authorId = (\d+);/)[1];
+    const authorName = $('script[type="text/javascript"]')
+        .text()
+        .match(/window\.authorName = "(.*)";/)[1];
+    const avatar = $('script[type="text/javascript"]')
+        .text()
+        .match(/avatar: "(.*?)",/)[1];
+    const introduce = $('script[type="text/javascript"]')
+        .text()
+        .match(/introduce: "(.*?)"\n/)[1];
+
+    const {
+        data: { data },
+    } = await got('https://blog.caixin.com/blog-api/post/posts', {
+        searchParams: {
+            page: 1,
+            size: ctx.query.limit ? parseInt(ctx.query.limit) : 20,
+            content: '',
+            authorId,
+            sort: 'publishTime',
+            direction: 'DESC',
+        },
+    });
+
+    const posts = data.map((item) => ({
+        title: item.title,
+        description: item.brief,
+        author: item.displayName,
+        link: item.guid.replace('http://', 'https://'),
+        pubDate: parseDate(item.publishTime, 'x'),
+    }));
+
+    const items = await Promise.all(posts.map((item) => parseBlogArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `财新博客 - ${authorName}`,
+        link,
+        description: introduce,
+        image: avatar,
+        item: items,
+    };
+};

package/lib/v2/caixin/category.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+    const column = ctx.params.column;
+    const url = `https://${column}.caixin.com/${category}`;
+    if (!isValidHost(column)) {
+        throw Error('Invalid column');
+    }
+
+    const response = await got(url);
+
+    const $ = cheerio.load(response.data);
+    const title = $('head title').text();
+    const entity = JSON.parse(
+        $('script')
+            .text()
+            .match(/var entity = (\{.*?\})/)[1]
+    );
+
+    const {
+        data: { datas: data },
+    } = await got('https://gateway.caixin.com/api/extapi/homeInterface.jsp', {
+        searchParams: {
+            subject: entity.id,
+            type: 0,
+            count: ctx.query.limit ? parseInt(ctx.query.limit) : 25,
+            picdim: '_266_177',
+            start: 0,
+        },
+    });
+
+    const list = data.map((item) => ({
+        title: item.desc,
+        description: item.summ,
+        link: item.link.replace('http://', 'https://'),
+        pubDate: timezone(parseDate(item.time), +8),
+        category: item.keyword.split(' '),
+        audio: item.audioUrl,
+        audio_image_url: item.pict.imgs[0].url,
+    }));
+
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title,
+        link: url,
+        description: '财新网 - 提供财经新闻及资讯服务',
+        item: items,
+    };
+};

package/lib/{routes → v2}/caixin/database.js

@@ -2,38 +2,38 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
     const rootUrl = 'https://database.caixin.com';
-    const currentUrl = `${rootUrl}/news`;
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
+    const currentUrl = `${rootUrl}/news/`;
+    const response = await got(currentUrl);
 
     const $ = cheerio.load(response.data);
 
     const list = $('h4 a')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             return {
                 title: item.text(),
-                link: item.attr('href'),
+                link: item.attr('href').replace('http://', 'https://'),
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
+                const detailResponse = await got(item.link);
                 const content = cheerio.load(detailResponse.data);
 
                 item.pubDate = timezone(parseDate(content('#pubtime_baidu').text()), +8);
-                item.description = `<p>${content('#subhead').html()}</p>`;
+                item.description = art(path.join(__dirname, 'templates/article.art'), {
+                    item,
+                    $: content,
+                });
+                item.author = content('.top-author').text();
 
                 return item;
             })

package/lib/v2/caixin/k.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const response = await got('https://k.caixin.com/app/v1/list', {
+        searchParams: {
+            productIdList: '8,28',
+            uid: '',
+            unit: 1,
+            name: '',
+            code: '',
+            deviceType: '',
+            device: '',
+            userTag: '',
+            p: 1,
+            c: 20,
+        },
+    });
+
+    const data = response.data.data.list;
+    const items = data.map((item) => {
+        const hasAudio = item.audio_url || Object.values(item.audios)[0];
+        return {
+            title: item.title,
+            description: item.text,
+            link: `http://k.caixin.com/web/detail_${item.oneline_news_code}`,
+            pubDate: parseDate(item.ts, 'x'),
+            author: '财新一线',
+            enclosure_url: hasAudio ? item.audio_url || Object.values(item.audios)[0] : undefined,
+            enclosure_type: hasAudio ? 'audio/mpeg' : undefined,
+        };
+    });
+
+    ctx.state.data = {
+        title: '财新网 - 财新一线新闻',
+        link: 'https://k.caixin.com/web/',
+        description: '财新网 - 财新一线新闻',
+        item: items,
+    };
+};

package/lib/v2/caixin/latest.js

@@ -0,0 +1,50 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const li_r = await got('https://www.caixin.com/2021-08-22/101758426.html');
+
+    const $ = cheerio.load(li_r.data);
+    const list = $('div.columnBox a[name="new_artical"]~ul.list li')
+        .toArray()
+        .map((li) => ({
+            link: $(li).find('a').first().attr('href').replace('http://', 'https://'),
+        }))
+        .filter((item) => !item.link.startsWith('https://fm.caixin.com/') && !item.link.startsWith('https://video.caixin.com/') && !item.link.startsWith('https://datanews.caixin.com/')); // content filter
+
+    const rss = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(`caixin:latest:${item.link}`, async () => {
+                const entry_r = await got(item.link);
+                const $ = cheerio.load(entry_r.data);
+                // title
+                const h1 = $('div#conTit h1').text();
+
+                // desc
+                const desc = art(path.join(__dirname, 'templates/article.art'), {
+                    item,
+                    $,
+                });
+
+                item.title = h1;
+                item.description = desc;
+                item.pubDate = timezone(parseDate($('#pubtime_baidu').text()), +8);
+                // prevent cache coliision with /caixin/article and /caixin/:column/:category
+                // since those have podcasts
+                item.guid = `caixin:latest:${item.link}`;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '财新网 - 最新文章',
+        link: 'https://www.caixin.com/',
+        item: rss,
+    };
+};

package/lib/v2/caixin/maintainer.js

@@ -0,0 +1,8 @@
+module.exports = {
+    '/article': ['EsuRt'],
+    '/blog/:column': ['Maecenas'],
+    '/database': ['nczitzk'],
+    '/k': ['boypt'],
+    '/latest': ['tpnonthealps'],
+    '/:column/:category': ['idealclover'],
+};

package/lib/v2/caixin/radar.js

@@ -0,0 +1,43 @@
+module.exports = {
+    'caixin.com': {
+        _name: '财新网',
+        '.': [
+            {
+                title: '新闻分类',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+            },
+            {
+                title: '首页新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+                source: ['/'],
+                target: '/caixin/article',
+            },
+            {
+                title: '最新文章',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+                source: ['/'],
+                target: '/caixin/latest',
+            },
+            {
+                title: '用户博客',
+                docs: 'https://docs.rsshub.app/blog.html#cai-xin-bo-ke',
+            },
+        ],
+        database: [
+            {
+                title: '财新数据通',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+                source: ['/news', '/'],
+                target: '/caixin/database',
+            },
+        ],
+        k: [
+            {
+                title: '财新一线',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+                source: ['/web', '/'],
+                target: '/caixin/database',
+            },
+        ],
+    },
+};

package/lib/v2/caixin/router.js

@@ -0,0 +1,8 @@
+module.exports = (router) => {
+    router.get('/article', require('./article'));
+    router.get('/blog/:column', require('./blog'));
+    router.get('/database', require('./database'));
+    router.get('/k', require('./k'));
+    router.get('/latest', require('./latest'));
+    router.get('/:column/:category', require('./category'));
+};

package/lib/v2/caixin/templates/article.art

@@ -0,0 +1,38 @@
+{{ if item.audio }}
+    <audio src="{{ item.audio }}" controls></audio><br>
+{{ /if }}
+{{ if $('.article .subhead').length }}
+    <blockquote><p>{{@ $('.article .subhead').html() }}</p></blockquote>
+    <br>
+{{ /if }}
+
+{{ if $('.article .media').length }}
+    {{@ $('.article .media').html() }}
+    <br>
+{{ /if }}
+
+{{ if $('.article .content_video').length }}
+    <% const video = $('script').text().match(/initPlayer\('(.*?)','(.*?)'\)/); %>
+    <% const videoUrl = video[1]; %>
+    <% const poster = video[2]; %>
+    <video controls preload="none" poster="{{ poster }}" src="{{ videoUrl }}"></video>
+    <br>
+{{ /if }}
+
+{{ if $('div#Main_Content_Val.text').length }}
+    {{@ $('div#Main_Content_Val.text').html() }}
+{{ else }}
+    {{ if item.summary }}
+        <blockquote><p>{{ item.summary }}</p></blockquote>
+        <br>
+    {{ /if }}
+    {{ if item.pics?.includes('#') }}
+        {{ each item.pics.split('#') pic }}
+            <img src="{{ pic }}">
+            <br>
+        {{ /each }}
+    {{ else }}
+        <img src="{{ item.pics }}">
+        <br>
+    {{ /if }}
+{{ /if }}

package/lib/v2/caixin/utils.js

@@ -0,0 +1,52 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const parseArticle = (item, tryGet) => {
+    if (new URL(item.link).hostname.match(/\.blog\.caixin\.com$/)) {
+        return parseBlogArticle(item, tryGet);
+    } else {
+        return tryGet(item.link, async () => {
+            const { data: response } = await got(item.link);
+
+            const $ = cheerio.load(response);
+
+            item.description = art(path.join(__dirname, 'templates/article.art'), {
+                item,
+                $,
+            });
+
+            if (item.audio) {
+                item.itunes_item_image = item.audio_image_url;
+                item.enclosure_url = item.audio;
+                item.enclosure_type = 'audio/mpeg';
+            }
+
+            return item;
+        });
+    }
+};
+
+const parseBlogArticle = (item, tryGet) =>
+    tryGet(item.link, async () => {
+        const response = await got(item.link);
+        const $ = cheerio.load(response.data);
+        const article = $('#the_content').removeAttr('style');
+        article.find('img').removeAttr('style');
+        article
+            .find('p')
+            // Non-breaking space U+00A0, ` ` in html
+            // element.children[0].data === $(element, article).text()
+            .filter((_, element) => element.children[0].data === String.fromCharCode(160))
+            .remove();
+
+        item.description = article.html();
+
+        return item;
+    });
+
+module.exports = {
+    parseArticle,
+    parseBlogArticle,
+};