rsshub 1.0.0-master.f961961 → 1.0.0-master.f992838

package/lib/v2/bloomberg/authors.js

@@ -0,0 +1,52 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const rssParser = require('@/utils/rss-parser');
+const { asyncPoolAll, parseArticle } = require('./utils');
+
+const parseAuthorNewsList = async (slug) => {
+    const baseURL = `https://www.bloomberg.com/authors/${slug}`;
+    const apiUrl = `https://www.bloomberg.com/lineup/api/lazy_load_author_stories?slug=${slug}&authorType=default&page=1`;
+    const resp = await got(apiUrl);
+    // Likely rate limited
+    if (!resp.data.html) {
+        return [];
+    }
+    const $ = cheerio.load(resp.data.html);
+    const articles = $('article.story-list-story');
+    return articles
+        .map((index, item) => {
+            item = $(item);
+            const headline = item.find('a.story-list-story__info__headline-link');
+            return {
+                title: headline.text(),
+                pubDate: item.attr('data-updated-at'),
+                guid: `bloomberg:${item.attr('data-id')}`,
+                link: new URL(headline.attr('href'), baseURL).href,
+            };
+        })
+        .get();
+};
+
+module.exports = async (ctx) => {
+    const { id, slug, source } = ctx.params;
+    const link = `https://www.bloomberg.com/authors/${id}/${slug}`;
+
+    let list = [];
+    if (!source || source === 'api') {
+        list = await parseAuthorNewsList(`${id}/${slug}`);
+    }
+    // Fallback to rss if api failed or requested by param
+    if (source === 'rss' || list.length === 0) {
+        list = (await rssParser.parseURL(`${link}.rss`)).items;
+    }
+
+    const item = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx));
+    const authorName = item.find((i) => i.author)?.author ?? 'Unknown';
+
+    ctx.state.data = {
+        title: `Bloomberg - ${authorName}`,
+        link,
+        language: 'en-us',
+        item,
+    };
+};

package/lib/v2/bloomberg/index.js

@@ -1,5 +1,4 @@
-const { rootUrl, parseNewsList, parseArticle } = require('./utils');
-const asyncPool = require('tiny-async-pool');
+const { rootUrl, asyncPoolAll, parseNewsList, parseArticle } = require('./utils');
 const site_title_mapping = {
     '/': 'News',
     bpol: 'Politics',
@@ -15,14 +14,6 @@ const site_title_mapping = {
     citylab: 'CityLab',
 };
 
-const asyncPoolAll = async (...args) => {
-    const results = [];
-    for await (const result of asyncPool(...args)) {
-        results.push(result);
-    }
-    return results;
-};
-
 module.exports = async (ctx) => {
     const { site } = ctx.params;
     const currentUrl = site ? `${rootUrl}/${site}/sitemap_news.xml` : `${rootUrl}/sitemap_news.xml`;

package/lib/v2/bloomberg/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/authors/:id/:slug/:source?': ['josh'],
     '/:site?': ['bigfei'],
     '/': ['bigfei'],
 };

package/lib/v2/bloomberg/radar.js

@@ -1,13 +1,19 @@
 module.exports = {
     'bloomberg.com': {
-        _name: 'bloomberg',
+        _name: 'Bloomberg',
         www: [
             {
-                title: 'Bloomberg',
-                docs: 'https://docs.rsshub.app/finance.html##bloomberg-news',
+                title: 'News',
+                docs: 'https://docs.rsshub.app/finance.html#bloomberg-news',
                 source: ['/:site', '/'],
                 target: '/bloomberg/:site?',
             },
+            {
+                title: 'Authors',
+                docs: 'https://docs.rsshub.app/finance.html#bloomberg',
+                source: ['/*/authors/:id/:slug', '/authors/:id/:slug'],
+                target: '/bloomberg/authors/:id/:slug',
+            },
         ],
     },
 };

package/lib/v2/bloomberg/router.js

@@ -1,4 +1,5 @@
 module.exports = function (router) {
+    router.get('/authors/:id/:slug/:source?', require('./authors'));
     router.get('/:site', require('./index'));
     router.get('/', require('./index'));
 };

package/lib/v2/bloomberg/utils.js

@@ -1,5 +1,6 @@
 const cheerio = require('cheerio');
 const path = require('path');
+const asyncPool = require('tiny-async-pool');
 
 const { parseDate } = require('@/utils/parse-date');
 const got = require('@/utils/got');
@@ -22,7 +23,7 @@ const apiEndpoints = {
     },
     videos: {
         url: 'https://www.bloomberg.com/news/videos/',
-        sel: '.footer-container + script',
+        sel: 'script',
     },
     newsletters: {
         url: 'https://www.bloomberg.com/news/newsletters/',
@@ -152,7 +153,9 @@ const parseAudioPage = async (res, api, item) => {
 };
 
 const parseVideoPage = async (res, api, item) => {
-    const json = cheerio.load(res.data)(api.sel).text().trim().replace('window.__PRELOADED_STATE__ = ', '').slice(0, -1);
+    const $ = cheerio.load(res.data);
+    const script = $(api.sel).filter((i, el) => $(el).text().includes('__PRELOADED_STATE__'));
+    const json = script.text().trim().replace('window.__PRELOADED_STATE__ = ', '').slice(0, -1);
     const article_json = JSON.parse(json);
     const video_story = article_json.video?.videoStory ?? article_json.quicktakeVideo?.videoStory;
     if (video_story) {
@@ -388,8 +391,17 @@ const processBody = async (body_html, story_json) => {
     return $.html();
 };
 
+const asyncPoolAll = async (...args) => {
+    const results = [];
+    for await (const result of asyncPool(...args)) {
+        results.push(result);
+    }
+    return results;
+};
+
 module.exports = {
     rootUrl,
+    asyncPoolAll,
     parseNewsList,
     parseArticle,
 };

package/lib/v2/btzj/index.js

@@ -4,10 +4,15 @@ const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const config = require('@/config').value;
+const allowDomain = ['2btjia.com', '88btbtt.com', 'btbtt15.com', 'btbtt20.com'];
 
 module.exports = async (ctx) => {
     let category = ctx.params.category ?? '';
-    let domain = ctx.query.domain ?? 'btbtt20.com';
+    let domain = ctx.query.domain ?? 'btbtt15.com';
+    if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(new URL(`http://${domain}/`).hostname)) {
+        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+    }
 
     if (category === 'base') {
         category = '';

package/lib/v2/bytes/bytes.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const currentURL = 'https://bytes.dev/archives';
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const resp = await got(currentURL);
+    const $ = cheerio.load(resp.data);
+    const text = $('script#__NEXT_DATA__').text();
+    const json = JSON.parse(text);
+    const posts = [json.props.pageProps.featuredPost].concat(json.props.pageProps.posts);
+    const items = posts.map((item) => ({
+        title: `Issue ${item.slug}`,
+        pubDate: parseDate(item.date),
+        description: item.title,
+        link: `/archives/${item.slug}`,
+    }));
+
+    ctx.state.data = {
+        title: 'bytes.dev',
+        description: 'Your weekly dose of JS',
+        link: currentURL,
+        item: items,
+    };
+};

package/lib/v2/bytes/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '': ['meixger'],
+    '/': ['meixger'],
+};

package/lib/v2/bytes/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'bytes.dev': {
+        _name: 'ui.dev',
+        '.': [
+            {
+                title: 'BYTES - Your weekly dose of JS',
+                docs: 'https://docs.rsshub.app/en/programming.html#ui-dev-bytes',
+                source: ['/archives', '/'],
+                target: '/bytes',
+            },
+        ],
+    },
+};

package/lib/v2/bytes/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./bytes'));
+};

package/lib/v2/caijing/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/roll': ['TonyRL'],
+};

package/lib/v2/caijing/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'caijing.com.cn': {
+        _name: '财经网',
+        roll: [
+            {
+                title: '滚动新闻',
+                docs: 'https://docs.rsshub.app/finance.html#cai-jing-wang',
+                source: ['/index1.html', '/'],
+                target: '/caijing/roll',
+            },
+        ],
+    },
+};

package/lib/v2/caijing/roll.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://roll.caijing.com.cn';
+    const response = await got(`${baseUrl}/ajax_lists.php`, {
+        searchParams: {
+            modelid: 0,
+            time: Math.random(),
+        },
+    });
+
+    const list = response.data.map((item) => ({
+        title: item.title,
+        link: item.url.replace('http://', 'https://'),
+        pubDate: timezone(parseDate(item.published, 'MM-DD HH:mm'), +8),
+        category: item.cat,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                item.author = $('.editor').text().trim() || $('#editor_baidu').text().trim().replace(/[()]/g, '');
+                item.description = $('.article-content').html();
+                item.category = [
+                    item.category,
+                    ...$('.news_keywords span')
+                        .toArray()
+                        .map((e) => $(e).text()),
+                ];
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '滚动新闻-财经网',
+        image: 'https://www.caijing.com.cn/favicon.ico',
+        link: response.url,
+        item: items,
+    };
+};

package/lib/v2/caijing/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/roll', require('./roll'));
+};

package/lib/v2/caixin/article.js

@@ -0,0 +1,27 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { data: response } = await got('https://mapiv5.caixin.com/m/api/getWapIndexListByPage');
+
+    const list = response.data.list.map((item) => ({
+        title: item.title,
+        description: item.summary,
+        author: item.author_name,
+        pubDate: parseDate(item.time, 'X'),
+        link: item.web_url,
+        pics: item.pics,
+        audio: item.cms_audio_url,
+        audio_image_url: item.audio_image_url,
+    }));
+
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: '财新网 - 首页',
+        link: 'https://www.caixin.com',
+        description: '财新网 - 首页',
+        item: items,
+    };
+};

package/lib/v2/caixin/blog.js

@@ -0,0 +1,58 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
+const { parseDate } = require('@/utils/parse-date');
+const { parseBlogArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { column } = ctx.params;
+    if (!isValidHost(column)) {
+        throw Error('Invalid column');
+    }
+    const link = `https://${column}.blog.caixin.com`;
+    const { data: response } = await got(link);
+    const $ = cheerio.load(response);
+    const authorId = $('script[type="text/javascript"]')
+        .text()
+        .match(/window\.authorId = (\d+);/)[1];
+    const authorName = $('script[type="text/javascript"]')
+        .text()
+        .match(/window\.authorName = "(.*)";/)[1];
+    const avatar = $('script[type="text/javascript"]')
+        .text()
+        .match(/avatar: "(.*?)",/)[1];
+    const introduce = $('script[type="text/javascript"]')
+        .text()
+        .match(/introduce: "(.*?)"\n/)[1];
+
+    const {
+        data: { data },
+    } = await got('https://blog.caixin.com/blog-api/post/posts', {
+        searchParams: {
+            page: 1,
+            size: ctx.query.limit ? parseInt(ctx.query.limit) : 20,
+            content: '',
+            authorId,
+            sort: 'publishTime',
+            direction: 'DESC',
+        },
+    });
+
+    const posts = data.map((item) => ({
+        title: item.title,
+        description: item.brief,
+        author: item.displayName,
+        link: item.guid.replace('http://', 'https://'),
+        pubDate: parseDate(item.publishTime, 'x'),
+    }));
+
+    const items = await Promise.all(posts.map((item) => parseBlogArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `财新博客 - ${authorName}`,
+        link,
+        description: introduce,
+        image: avatar,
+        item: items,
+    };
+};

package/lib/v2/caixin/category.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+    const column = ctx.params.column;
+    const url = `https://${column}.caixin.com/${category}`;
+    if (!isValidHost(column)) {
+        throw Error('Invalid column');
+    }
+
+    const response = await got(url);
+
+    const $ = cheerio.load(response.data);
+    const title = $('head title').text();
+    const entity = JSON.parse(
+        $('script')
+            .text()
+            .match(/var entity = (\{.*?\})/)[1]
+    );
+
+    const {
+        data: { datas: data },
+    } = await got('https://gateway.caixin.com/api/extapi/homeInterface.jsp', {
+        searchParams: {
+            subject: entity.id,
+            type: 0,
+            count: ctx.query.limit ? parseInt(ctx.query.limit) : 25,
+            picdim: '_266_177',
+            start: 0,
+        },
+    });
+
+    const list = data.map((item) => ({
+        title: item.desc,
+        description: item.summ,
+        link: item.link.replace('http://', 'https://'),
+        pubDate: timezone(parseDate(item.time), +8),
+        category: item.keyword.split(' '),
+        audio: item.audioUrl,
+        audio_image_url: item.pict.imgs[0].url,
+    }));
+
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title,
+        link: url,
+        description: '财新网 - 提供财经新闻及资讯服务',
+        item: items,
+    };
+};

package/lib/{routes → v2}/caixin/database.js

@@ -2,38 +2,38 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
     const rootUrl = 'https://database.caixin.com';
-    const currentUrl = `${rootUrl}/news`;
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
+    const currentUrl = `${rootUrl}/news/`;
+    const response = await got(currentUrl);
 
     const $ = cheerio.load(response.data);
 
     const list = $('h4 a')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item);
             return {
                 title: item.text(),
-                link: item.attr('href'),
+                link: item.attr('href').replace('http://', 'https://'),
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
+                const detailResponse = await got(item.link);
                 const content = cheerio.load(detailResponse.data);
 
                 item.pubDate = timezone(parseDate(content('#pubtime_baidu').text()), +8);
-                item.description = `<p>${content('#subhead').html()}</p>`;
+                item.description = art(path.join(__dirname, 'templates/article.art'), {
+                    item,
+                    $: content,
+                });
+                item.author = content('.top-author').text();
 
                 return item;
             })

package/lib/v2/caixin/k.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const response = await got('https://k.caixin.com/app/v1/list', {
+        searchParams: {
+            productIdList: '8,28',
+            uid: '',
+            unit: 1,
+            name: '',
+            code: '',
+            deviceType: '',
+            device: '',
+            userTag: '',
+            p: 1,
+            c: 20,
+        },
+    });
+
+    const data = response.data.data.list;
+    const items = data.map((item) => {
+        const hasAudio = item.audio_url || Object.values(item.audios)[0];
+        return {
+            title: item.title,
+            description: item.text,
+            link: `http://k.caixin.com/web/detail_${item.oneline_news_code}`,
+            pubDate: parseDate(item.ts, 'x'),
+            author: '财新一线',
+            enclosure_url: hasAudio ? item.audio_url || Object.values(item.audios)[0] : undefined,
+            enclosure_type: hasAudio ? 'audio/mpeg' : undefined,
+        };
+    });
+
+    ctx.state.data = {
+        title: '财新网 - 财新一线新闻',
+        link: 'https://k.caixin.com/web/',
+        description: '财新网 - 财新一线新闻',
+        item: items,
+    };
+};

package/lib/v2/caixin/latest.js

@@ -0,0 +1,50 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const li_r = await got('https://www.caixin.com/2021-08-22/101758426.html');
+
+    const $ = cheerio.load(li_r.data);
+    const list = $('div.columnBox a[name="new_artical"]~ul.list li')
+        .toArray()
+        .map((li) => ({
+            link: $(li).find('a').first().attr('href').replace('http://', 'https://'),
+        }))
+        .filter((item) => !item.link.startsWith('https://fm.caixin.com/') && !item.link.startsWith('https://video.caixin.com/') && !item.link.startsWith('https://datanews.caixin.com/')); // content filter
+
+    const rss = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(`caixin:latest:${item.link}`, async () => {
+                const entry_r = await got(item.link);
+                const $ = cheerio.load(entry_r.data);
+                // title
+                const h1 = $('div#conTit h1').text();
+
+                // desc
+                const desc = art(path.join(__dirname, 'templates/article.art'), {
+                    item,
+                    $,
+                });
+
+                item.title = h1;
+                item.description = desc;
+                item.pubDate = timezone(parseDate($('#pubtime_baidu').text()), +8);
+                // prevent cache coliision with /caixin/article and /caixin/:column/:category
+                // since those have podcasts
+                item.guid = `caixin:latest:${item.link}`;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '财新网 - 最新文章',
+        link: 'https://www.caixin.com/',
+        item: rss,
+    };
+};

package/lib/v2/caixin/maintainer.js

@@ -0,0 +1,8 @@
+module.exports = {
+    '/article': ['EsuRt'],
+    '/blog/:column': ['Maecenas'],
+    '/database': ['nczitzk'],
+    '/k': ['boypt'],
+    '/latest': ['tpnonthealps'],
+    '/:column/:category': ['idealclover'],
+};

package/lib/v2/caixin/radar.js

@@ -0,0 +1,43 @@
+module.exports = {
+    'caixin.com': {
+        _name: '财新网',
+        '.': [
+            {
+                title: '新闻分类',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+            },
+            {
+                title: '首页新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+                source: ['/'],
+                target: '/caixin/article',
+            },
+            {
+                title: '最新文章',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+                source: ['/'],
+                target: '/caixin/latest',
+            },
+            {
+                title: '用户博客',
+                docs: 'https://docs.rsshub.app/blog.html#cai-xin-bo-ke',
+            },
+        ],
+        database: [
+            {
+                title: '财新数据通',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+                source: ['/news', '/'],
+                target: '/caixin/database',
+            },
+        ],
+        k: [
+            {
+                title: '财新一线',
+                docs: 'https://docs.rsshub.app/traditional-media.html#cai-xin-wang',
+                source: ['/web', '/'],
+                target: '/caixin/database',
+            },
+        ],
+    },
+};

package/lib/v2/caixin/router.js

@@ -0,0 +1,8 @@
+module.exports = (router) => {
+    router.get('/article', require('./article'));
+    router.get('/blog/:column', require('./blog'));
+    router.get('/database', require('./database'));
+    router.get('/k', require('./k'));
+    router.get('/latest', require('./latest'));
+    router.get('/:column/:category', require('./category'));
+};