rsshub 1.0.0-master.fe50f9c → 1.0.0-master.fe560d2

package/lib/v2/bloomberg/index.js

@@ -0,0 +1,37 @@
+const { rootUrl, parseNewsList, parseArticle } = require('./utils');
+const asyncPool = require('tiny-async-pool');
+const site_title_mapping = {
+    '/': 'News',
+    bpol: 'Politics',
+    bbiz: 'Business',
+    markets: 'Markets',
+    technology: 'Technology',
+    green: 'Green',
+    wealth: 'Wealth',
+    pursuits: 'Pursuits',
+    bview: 'Opinion',
+    equality: 'Equality',
+    businessweek: 'Businessweek',
+    citylab: 'CityLab',
+};
+
+const asyncPoolAll = async (...args) => {
+    const results = [];
+    for await (const result of asyncPool(...args)) {
+        results.push(result);
+    }
+    return results;
+};
+
+module.exports = async (ctx) => {
+    const { site } = ctx.params;
+    const currentUrl = site ? `${rootUrl}/${site}/sitemap_news.xml` : `${rootUrl}/sitemap_news.xml`;
+
+    const list = await parseNewsList(currentUrl, ctx);
+    const items = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx));
+    ctx.state.data = {
+        title: `Bloomberg - ${site_title_mapping[site ?? '/']}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/bloomberg/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/:site?': ['bigfei'],
+    '/': ['bigfei'],
+};

package/lib/v2/bloomberg/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'bloomberg.com': {
+        _name: 'bloomberg',
+        www: [
+            {
+                title: 'Bloomberg',
+                docs: 'https://docs.rsshub.app/finance.html##bloomberg-news',
+                source: ['/:site', '/'],
+                target: '/bloomberg/:site?',
+            },
+        ],
+    },
+};

package/lib/v2/bloomberg/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/:site', require('./index'));
+    router.get('/', require('./index'));
+};

package/lib/v2/bloomberg/templates/audio_media.art

@@ -0,0 +1,13 @@
+<figure>
+    <div>
+        <img src="{{ img }}">
+        <audio controls>
+            <source src="{{ src }}" type="audio/mpeg">
+            Your browser does not support the audio element.
+        </audio>
+    </div>
+    <figcaption>
+        <div class="caption">{{@ caption }}</div>
+        <div class="credit">{{@ credit }}</div>
+    </figcaption>
+</figure>

package/lib/v2/bloomberg/templates/image_figure.art

@@ -0,0 +1,9 @@
+<figure>
+    <img src="{{ src }}" alt="{{ alt }}" loading="lazy" style="display:block; margin-left:auto; margin-right:auto;" width:100%;>
+    {{if caption}}
+    <figcaption>
+        <div class="caption">{{@ caption }}</div>
+        <div class="credit">{{@ credit }}</div>
+    </figcaption>
+    {{/if}}
+</figure>

package/lib/v2/bloomberg/templates/lede_media.art

@@ -0,0 +1,14 @@
+{{if (media.kind =='image') }}
+<figure>
+    <img src="{{ media.src }}" alt="{{ media.description }}" loading="lazy" style="display:block; margin-left:auto; margin-right:auto;" width:100%;>
+    {{if media.caption}}
+    <figcaption>
+        <div class="caption">{{@ media.caption }}</div>
+        <div class="credit">{{@ media.credit }}</div>
+    </figcaption>
+    {{/if}}
+</figure>
+{{/if}}
+{{if (media.kind =='video') }}
+{{include './video_media.art' media.video}}
+{{/if}}

package/lib/v2/bloomberg/templates/video_media.art

@@ -0,0 +1,25 @@
+<figure>
+    <video controls
+        playsinline="true"
+        webkit-playsinline="true"
+        x5-playsinline="true"
+        x5-video-player-type="h5"
+        x5-video-orientation="landscape|portrait"
+        x5-video-player-fullscreen="true"
+        x-webkit-airplay="allow"
+        preload="metadata"
+        poster="{{ coverUrl }}">
+            {{if stream }}
+            <source src="{{ stream }}" type="application/x-mpegURL">
+            {{/if}}
+            {{if mp4 }}
+            <source src="{{ mp4 }}" type="video/mp4"/>
+            {{/if}}
+
+  </video>
+    {{if caption}}
+    <figcaption>
+        <div class="caption">{{@ caption }}</div>
+    </figcaption>
+    {{/if}}
+</figure>

package/lib/v2/bloomberg/utils.js

@@ -0,0 +1,395 @@
+const cheerio = require('cheerio');
+const path = require('path');
+
+const { parseDate } = require('@/utils/parse-date');
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+
+const rootUrl = 'https://www.bloomberg.com/feeds';
+const sel = 'script[data-component-props="ArticleBody"], script[data-component-props="FeatureBody"]';
+const apiEndpoints = {
+    articles: {
+        url: 'https://www.bloomberg.com/javelin/api/foundation_transporter/',
+        sel,
+    },
+    features: {
+        url: 'https://www.bloomberg.com/javelin/api/foundation_feature_transporter/',
+        sel,
+    },
+    audio: {
+        url: 'https://www.bloomberg.com/news/audio/',
+        sel: 'script#__NEXT_DATA__',
+    },
+    videos: {
+        url: 'https://www.bloomberg.com/news/videos/',
+        sel: '.footer-container + script',
+    },
+    newsletters: {
+        url: 'https://www.bloomberg.com/news/newsletters/',
+        sel,
+    },
+    'photo-essays': {
+        url: 'https://www.bloomberg.com/javelin/api/photo-essay_transporter/',
+        sel: 'script[type = "application/json"][data-component-props]',
+    },
+    'features/': {
+        url: 'https://www.bloomberg.com/features/',
+        sel: 'script#__SSR_DATA__',
+    },
+};
+const headers = {
+    accept: 'application/json',
+    'cache-control': 'no-cache',
+    referer: 'https://www.bloomberg.com',
+};
+
+const pageTypeRegex1 = /\/(?<page>[\w-]*?)\/(?<link>\d{4}-\d{2}-\d{2}\/.*)/;
+const pageTypeRegex2 = /(?<!news|politics)\/(?<page>features\/|graphics\/)(?<link>.*)/;
+const regex = [pageTypeRegex1, pageTypeRegex2];
+
+const capRegex = /<p>|<\/p>/g;
+const emptyRegex = /<p\b[^>]*>(&nbsp;|\s)<\/p>/g;
+
+const parseNewsList = async (url, ctx) => {
+    const resp = await got(url);
+    const $ = cheerio.load(resp.data, {
+        xml: {
+            xmlMode: true,
+        },
+    });
+    const urls = $('urlset url');
+    return urls
+        .toArray()
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
+        .map((u) => {
+            u = $(u);
+            const item = {
+                title: u.find('news\\:title').text(),
+                link: u.find('loc').text(),
+                pubDate: parseDate(u.find('news\\:publication_date').text()),
+            };
+            return item;
+        });
+};
+
+const parseArticle = async (item, ctx) =>
+    await ctx.cache.tryGet(item.link, async () => {
+        const group = regex
+            .map((r) => r.exec(item.link))
+            .filter((e) => e && e.groups)
+            .map((a) => a && a.groups)[0];
+        if (group) {
+            const { page, link } = group;
+
+            if (apiEndpoints[page]) {
+                const api = apiEndpoints[page];
+                const apiUrl = `${api.url}${link}`;
+                let res;
+                try {
+                    res = await got(apiUrl, { headers });
+                } catch (err) {
+                    // fallback
+                    if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
+                        try {
+                            res = await got(item.link, { headers });
+                        } catch (err) {
+                            // return the default one
+                            return {
+                                title: item.title,
+                                link: item.link,
+                                pubDate: item.pubDate,
+                            };
+                        }
+                    }
+                }
+
+                // Blocked by PX3, return the default
+                const redirectUrls = res.redirectUrls.map(String);
+                if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) {
+                    return {
+                        title: item.title,
+                        link: item.link,
+                        pubDate: item.pubDate,
+                    };
+                }
+
+                switch (page) {
+                    case 'audio':
+                        return await parseAudioPage(res, api, item);
+                    case 'videos':
+                        return await parseVideoPage(res, api, item);
+                    case 'photo-essays':
+                        return await parsePhotoEssaysPage(res, api, item);
+                    case 'features/': // single features page
+                        return await parseFeaturePage(res, api, item);
+                    default:
+                        return await parseOtherPage(res, api, item);
+                }
+            }
+        }
+        return item;
+    });
+
+const parseAudioPage = async (res, api, item) => {
+    const audio_json = JSON.parse(cheerio.load(res.data)(api.sel).html()).props.pageProps;
+    const episode = audio_json.episode;
+    const rss_item = {
+        title: episode.title || item.title,
+        link: audio_json.pageInfo.canonicalUrl || item.link,
+        guid: `bloomberg:${episode.id}`,
+        description: (await processBody(episode.articleBody, audio_json)).replaceAll(emptyRegex, ''),
+        pubDate: parseDate(episode.publishedAt) || item.pubDate,
+        author: audio_json.hero.showTitle,
+        media: {
+            content: { url: episode.image },
+            thumbnails: { url: episode.image },
+        },
+        enclosure_type: 'audio/mpeg',
+        enclosure_url: episode.url,
+        itunes_item_image: episode.image || audio_json.pageInfo.image.url,
+    };
+    return rss_item;
+};
+
+const parseVideoPage = async (res, api, item) => {
+    const json = cheerio.load(res.data)(api.sel).text().trim().replace('window.__PRELOADED_STATE__ = ', '').slice(0, -1);
+    const article_json = JSON.parse(json);
+    const video_story = article_json.video?.videoStory ?? article_json.quicktakeVideo?.videoStory;
+    if (video_story) {
+        const desc = await processVideo(video_story.video.bmmrId, video_story.summary.html.replaceAll(emptyRegex, ''));
+        const rss_item = {
+            title: video_story.headline.text || item.title,
+            link: video_story.url || item.link,
+            guid: `bloomberg:${video_story.id}`,
+            description: art(path.join(__dirname, 'templates/video_media.art'), desc),
+            pubDate: parseDate(video_story.publishedAt) || item.pubDate,
+            media: {
+                content: { url: video_story.video?.thumbnail.url || '' },
+                thumbnails: { url: video_story.video?.thumbnail.url || '' },
+            },
+            category: desc.keywords ?? [],
+        };
+        return rss_item;
+    }
+    return item;
+};
+
+const parsePhotoEssaysPage = async (res, api, item) => {
+    const $ = cheerio.load(res.data.html);
+    const article_json = $(api.sel)
+        .toArray()
+        .map((e) => JSON.parse($(e).html()))
+        .reduce((pv, cv) => ({ ...pv, ...cv }), {});
+    const rss_item = {
+        title: article_json.headline || item.title,
+        link: article_json.canonical || item.link,
+        guid: `bloomberg:${article_json.id}`,
+        description: (await processBody(article_json.body, article_json)).replaceAll(emptyRegex, ''),
+        pubDate: item.pubDate,
+        author: article_json.authors?.map((a) => a.name).join(', ') ?? [],
+    };
+    return rss_item;
+};
+
+const parseFeaturePage = async (res, api, item) => {
+    const json = cheerio.load(res.data)(api.sel).text().trim().split('\n')[0].replace('var ds__data = ', '').slice(0, -1);
+    const article_json = JSON.parse(JSON.parse(json));
+    const meta = article_json.meta;
+
+    const desc = article_json.blocks.map(async (b) => {
+        if (b.type === 'Paragraph') {
+            return b.props.text.html;
+        }
+        if (b.type === 'ImageBlock') {
+            const image = { src: b.props?.src || '', alt: b.props?.alt || '', caption: b.props?.caption || '', credit: b.props?.credit || '' };
+            return art(path.join(__dirname, 'templates/image_figure.art'), image);
+        }
+        if (b.type === 'Lede') {
+            return await processLedeMedia(b);
+        }
+        return '';
+    });
+    const rss_item = {
+        title: meta.title || item.title,
+        link: item.link,
+        description: (await Promise.all(desc)).join(''),
+        pubDate: parseDate(meta.pubDateTime) || item.pubDate,
+        author: meta.authors?.map((a) => a.name).join(', ') ?? [],
+        media: {
+            content: { url: meta.socialImage?.url ?? '' },
+            thumbnails: { url: meta.socialImage?.url ?? '' },
+        },
+        category:
+            meta.keywords
+                .split(',')
+                .map((e) => e.trim())
+                .filter((e) => !!e) ?? [],
+    };
+    return rss_item;
+};
+
+const parseOtherPage = async function (res, api, item) {
+    const article_json = JSON.parse(
+        cheerio
+            .load(res.data.html ?? res.data)(api.sel)
+            .html()
+    );
+    const story_json = article_json.story;
+    const body_html = story_json.body;
+    const media_img = story_json.ledeImageUrl || Object.values(story_json.imageAttachments ?? {})[0]?.baseUrl;
+
+    const rss_item = {
+        title: story_json.textHeadline || item.title,
+        link: story_json.canonical || item.link,
+        guid: `bloomberg:${story_json.id}`,
+        description: (processHeadline(story_json) + (await processLedeMedia(story_json)) + (await processBody(body_html, story_json))).replaceAll(emptyRegex, ''),
+        pubDate: parseDate(story_json.publishedAt) || item.pubDate,
+        author: story_json.authors?.map((a) => a.name).join(', ') ?? [],
+        category: story_json.mostRelevantTags ?? [],
+        media: {
+            content: { url: media_img },
+            thumbnails: { url: media_img },
+        },
+    };
+    return rss_item;
+};
+
+const processHeadline = (story_json) => {
+    const dek = story_json.dek || '';
+    const abs = story_json.abstract?.map((a) => `<li>${a}</li>`).join('');
+    return abs ? dek + `<ul>${abs}</ul>` : dek;
+};
+
+const processLedeMedia = async (story_json) => {
+    if (story_json.ledeMediaKind) {
+        const kind = story_json.ledeMediaKind;
+
+        const media = {
+            kind: story_json.ledeMediaKind,
+            caption: story_json.ledeCaption?.replaceAll(capRegex, '') ?? '',
+            description: story_json.ledeDescription?.replaceAll(capRegex, '') ?? '',
+            credit: story_json.ledeCredit?.replaceAll(capRegex, '') ?? '',
+            src: story_json.ledeImageUrl,
+            video: kind === 'video' && (await processVideo(story_json.ledeAttachment.bmmrId)),
+        };
+        return art(path.join(__dirname, 'templates/lede_media.art'), { media });
+    } else if (story_json.imageAttachments) {
+        const attachment = Object.values(story_json.imageAttachments)[0];
+        if (attachment) {
+            const image = {
+                src: attachment.baseUrl,
+                alt: attachment.alt,
+            };
+            return art(path.join(__dirname, 'templates/image_figure.art'), image);
+        }
+        return '';
+    } else if (story_json.type === 'Lede') {
+        const props = story_json.props;
+
+        const media = {
+            kind: props.media,
+            caption: props.caption?.replaceAll(capRegex, '') ?? '',
+            description: props.dek?.replaceAll(capRegex, '') ?? '',
+            credit: props.credit?.replaceAll(capRegex, '') ?? '',
+            src: props.url,
+        };
+        return art(path.join(__dirname, 'templates/lede_media.art'), { media });
+    }
+};
+
+const processVideo = async (bmmrId, summary) => {
+    const api = `https://www.bloomberg.com/multimedia/api/embed?id=${bmmrId}`;
+    const res = await got(api, { headers });
+
+    // Blocked by PX3, return the default
+    const redirectUrls = res.redirectUrls.map(String);
+    if (redirectUrls.some((r) => new URL(r).pathname === '/tosv2.html')) {
+        return {
+            stream: '',
+            mp4: '',
+            coverUrl: '',
+            caption: summary,
+        };
+    }
+
+    if (res.data) {
+        const video_json = res.data;
+        return {
+            stream: video_json.streams ? video_json.streams[0]?.url : '',
+            mp4: video_json.downloadURLs ? video_json.downloadURLs['600'] : '',
+            coverUrl: video_json.thumbnail?.baseUrl ?? '',
+            caption: video_json.description || video_json.title || summary,
+        };
+    }
+    return {};
+};
+
+const processBody = async (body_html, story_json) => {
+    const removeSel = ['meta', 'script', '*[class$="-footnotes"]', '*[class$="for-you"]', '*[class$="-newsletter"]', '*[class$="page-ad"]', '*[class$="-recirc"]', '*[data-ad-placeholder="Advertisement"]'];
+
+    const $ = cheerio.load(body_html);
+    removeSel.forEach((sel) => $(sel).remove());
+    $('.paywall').removeAttr('class');
+
+    // Asynchronous iteration intentionally
+    // https://github.com/eslint/eslint/blob/8a159686f9d497262d573dd601855ce28362199b/tests/lib/rules/no-await-in-loop.js#L50
+    for await (const e of $('figure')) {
+        const imageType = $(e).data('image-type');
+        const type = $(e).data('type');
+
+        let new_figure = '';
+        if (imageType === 'audio') {
+            let audio = {};
+            if (story_json.audios) {
+                const attachment = story_json.audios.find((a) => a.id.toString() === $(e).data('id').toString());
+                audio = {
+                    img: attachment.image?.url || $(e).find('img').attr('src'),
+                    src: attachment.url || $(e).find('audio source').attr('src'),
+                    caption: $(e).find('[class$="text"]').html()?.trim() ?? '',
+                    credit: $(e).find('[class$="credit"]').html()?.trim() ?? '',
+                };
+            }
+            if (story_json.episode) {
+                const episode = story_json.episode;
+                audio = {
+                    src: episode.url,
+                    img: episode.image || story_json.pageInfo.image.url,
+
+                    caption: episode.description || ($(e).find('[class$="text"]').html()?.trim() ?? ''),
+                    credit: (episode.credits.map((c) => c.name).join(', ') ?? []) || ($(e).find('[class$="credit"]').html()?.trim() ?? ''),
+                };
+            }
+            new_figure = art(path.join(__dirname, 'templates/audio_media.art'), audio);
+        } else if (imageType === 'video') {
+            if (story_json.videoAttachments) {
+                const attachment = story_json.videoAttachments[$(e).data('id')];
+                const video = await processVideo(attachment.bmmrId);
+                new_figure = art(path.join(__dirname, 'templates/video_media.art'), video);
+            }
+        } else if (imageType === 'photo' || imageType === 'image' || type === 'image') {
+            let src, alt;
+            if (story_json.imageAttachments) {
+                const attachment = story_json.imageAttachments[$(e).data('id')];
+                alt = attachment?.alt || $(e).find('img').attr('alt')?.trim();
+                src = attachment?.baseUrl;
+            } else {
+                alt = $(e).find('img').attr('alt').trim();
+                src = $(e).find('img').data('native-src');
+            }
+            const caption = $(e).find('[class$="text"], .caption, .photo-essay__text').html()?.trim() ?? '';
+            const credit = $(e).find('[class$="credit"], .credit, .photo-essay__source').html()?.trim() ?? '';
+            const image = { src, alt, caption, credit };
+            new_figure = art(path.join(__dirname, 'templates/image_figure.art'), image);
+        }
+        $(new_figure).insertAfter(e);
+        $(e).remove();
+    }
+
+    return $.html();
+};
+
+module.exports = {
+    rootUrl,
+    parseNewsList,
+    parseArticle,
+};

package/lib/v2/bytes/bytes.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const currentURL = 'https://bytes.dev/archives';
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const resp = await got(currentURL);
+    const $ = cheerio.load(resp.data);
+    const text = $('script#__NEXT_DATA__').text();
+    const json = JSON.parse(text);
+    const posts = [json.props.pageProps.featuredPost].concat(json.props.pageProps.posts);
+    const items = posts.map((item) => ({
+        title: `Issue ${item.slug}`,
+        pubDate: parseDate(item.date),
+        description: item.title,
+        link: `/archives/${item.slug}`,
+    }));
+
+    ctx.state.data = {
+        title: 'bytes.dev',
+        description: 'Your weekly dose of JS',
+        link: currentURL,
+        item: items,
+    };
+};

package/lib/v2/bytes/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '': ['meixger'],
+    '/': ['meixger'],
+};

package/lib/v2/bytes/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'bytes.dev': {
+        _name: 'ui.dev',
+        '.': [
+            {
+                title: 'BYTES - Your weekly dose of JS',
+                docs: 'https://docs.rsshub.app/en/programming.html#ui-dev-bytes',
+                source: ['/archives', '/'],
+                target: '/bytes',
+            },
+        ],
+    },
+};

package/lib/v2/bytes/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./bytes'));
+};

package/lib/v2/caijing/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/roll': ['TonyRL'],
+};

package/lib/v2/caijing/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'caijing.com.cn': {
+        _name: '财经网',
+        roll: [
+            {
+                title: '滚动新闻',
+                docs: 'https://docs.rsshub.app/finance.html#cai-jing-wang',
+                source: ['/index1.html', '/'],
+                target: '/caijing/roll',
+            },
+        ],
+    },
+};

package/lib/v2/caijing/roll.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://roll.caijing.com.cn';
+    const response = await got(`${baseUrl}/ajax_lists.php`, {
+        searchParams: {
+            modelid: 0,
+            time: Math.random(),
+        },
+    });
+
+    const list = response.data.map((item) => ({
+        title: item.title,
+        link: item.url.replace('http://', 'https://'),
+        pubDate: timezone(parseDate(item.published, 'MM-DD HH:mm'), +8),
+        category: item.cat,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                item.author = $('.editor').text().trim() || $('#editor_baidu').text().trim().replace(/[()]/g, '');
+                item.description = $('.article-content').html();
+                item.category = [
+                    item.category,
+                    ...$('.news_keywords span')
+                        .toArray()
+                        .map((e) => $(e).text()),
+                ];
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '滚动新闻-财经网',
+        image: 'https://www.caijing.com.cn/favicon.ico',
+        link: response.url,
+        item: items,
+    };
+};

package/lib/v2/caijing/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/roll', require('./roll'));
+};

package/lib/v2/cdzjryb/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/zw/projectList': ['TonyRL'],
+};