rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/lib/v2/baidu/radar.js

@@ -56,5 +56,13 @@ module.exports = {
                 },
             },
         ],
+        top: [
+            {
+                title: '热搜榜单',
+                docs: 'https://docs.rsshub.app/other.html#bai-du-re-sou',
+                source: ['/board'],
+                target: (_, url) => `/baidu/top/${new URL(url).searchParams.get('tab')}`,
+            },
+        ],
     },
 };

package/lib/v2/baidu/router.js

@@ -5,4 +5,5 @@ module.exports = (router) => {
     router.get('/tieba/post/:id', require('./tieba/post'));
     router.get('/tieba/post/lz/:id', require('./tieba/post'));
     router.get('/tieba/user/:uid', require('./tieba/user'));
+    router.get('/top/:board?', require('./top'));
 };

package/lib/v2/baidu/templates/top.art

@@ -0,0 +1,9 @@
+{{ if item.img }}
+<img src="{{ item.img }}"><br>
+{{ /if }}
+{{ if item.show }}
+    {{ each item.show s }}
+        {{ s }}<br>
+    {{ /each }}
+{{ /if }}
+{{ item.desc }}

package/lib/v2/baidu/top.js

@@ -0,0 +1,34 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { board = 'realtime' } = ctx.params;
+    const link = `https://top.baidu.com/board?tab=${board}`;
+    const { data: response } = await got(link);
+
+    const $ = cheerio.load(response);
+
+    const { data } = JSON.parse(
+        $('#sanRoot')
+            .contents()
+            .filter((e) => e.nodeType === 8)
+            .prevObject[0].data.match(/s-data:(.*)/)[1]
+    );
+
+    const items = data.cards[0].content.map((item) => ({
+        title: item.word,
+        description: art(path.join(__dirname, 'templates/top.art'), {
+            item,
+        }),
+        link: item.rawUrl,
+    }));
+
+    ctx.state.data = {
+        title: `${data.curBoardName} - 百度热搜`,
+        description: $('meta[name="description"]').attr('content'),
+        link,
+        item: items,
+    };
+};

package/lib/v2/bendibao/news.js

@@ -2,9 +2,13 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const city = ctx.params.city;
+    if (!isValidHost(city)) {
+        throw Error('Invalid city');
+    }
 
     const rootUrl = `http://${city}.bendibao.com`;
 

package/lib/v2/bioone/featured.js

@@ -4,9 +4,10 @@ const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const rootUrl = 'https://bioone.org';
-    const response = await got({
-        method: 'get',
-        url: rootUrl,
+    const response = await got(rootUrl, {
+        https: {
+            rejectUnauthorized: false,
+        },
     });
 
     const $ = cheerio.load(response.data);
@@ -16,20 +17,21 @@ module.exports = async (ctx) => {
         .toArray()
         .map((item) => {
             item = $(item);
-            const link = item.attr('href');
+            const link = item.attr('href').split('?')[0];
 
             return {
                 title: item.text(),
-                link: link.indexOf('http') < 0 ? `${rootUrl}${link}` : link,
+                link: !link.includes('http') ? `${rootUrl}${link}` : link,
             };
         });
 
     items = await Promise.all(
         items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
+                const detailResponse = await got(item.link, {
+                    https: {
+                        rejectUnauthorized: false,
+                    },
                 });
                 const content = cheerio.load(detailResponse.data);
 

package/lib/v2/bioone/journal.js

@@ -7,15 +7,16 @@ module.exports = async (ctx) => {
 
     const rootUrl = 'https://bioone.org';
     const currentUrl = `${rootUrl}/journals/${journal}/current`;
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
+    const response = await got(currentUrl, {
+        https: {
+            rejectUnauthorized: false,
+        },
     });
 
     const $ = cheerio.load(response.data);
 
     let items = $('.TOCLineItemBoldText')
-        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 10)
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20)
         .toArray()
         .map((item) => {
             item = $(item).parent();
@@ -29,9 +30,10 @@ module.exports = async (ctx) => {
         items.map(
             async (item) =>
                 await ctx.cache.tryGet(item.link, async () => {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
+                    const detailResponse = await got(item.link, {
+                        https: {
+                            rejectUnauthorized: false,
+                        },
                     });
 
                     const content = cheerio.load(detailResponse.data);
@@ -51,7 +53,8 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: `${response.data.match(/'journalpaper', '(.*)'\)\\" class=\\"/)[1]} - BioOne`,
+        title: `${$('.panel-body .row a').first().text()} - BioOne`,
+        description: $('.panel-body .row text').first().text(),
         link: currentUrl,
         item: items,
     };

package/lib/v2/biquge/index.js

@@ -3,10 +3,31 @@ const cheerio = require('cheerio');
 const iconv = require('iconv-lite');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const allowHost = [
+    'www.xbiquwx.la',
+    'www.biqu5200.net',
+    'www.xbiquge.so',
+    'www.biqugeu.net',
+    'www.b520.cc',
+    'www.ahfgb.com',
+    'www.ibiquge.la',
+    'www.biquge.tv',
+    'www.bswtan.com',
+    'www.biquge.co',
+    'www.bqzhh.com',
+    'www.biqugse.com',
+    'www.ibiquge.info',
+    'www.ishuquge.com',
+    'www.mayiwxw.com',
+];
 
 module.exports = async (ctx) => {
     const rootUrl = ctx.path.split('/').slice(1, 4).join('/');
     const currentUrl = ctx.path.slice(1);
+    if (!config.feature.allow_user_supply_unsafe_domain && !allowHost.includes(new URL(rootUrl).hostname)) {
+        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+    }
 
     const response = await got({
         method: 'get',

package/lib/v2/bitmovin/blog.js

@@ -1,33 +1,27 @@
 const got = require('@/utils/got');
-const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 
+const baseUrl = 'https://bitmovin.com';
+
 module.exports = async (ctx) => {
-    const rootUrl = 'https://bitmovin.com/blog/';
-    const response = await got({
-        method: 'get',
-        url: rootUrl,
+    const apiUrl = `${baseUrl}/wp-json/wp/v2`;
+    const { data } = await got(`${apiUrl}/posts`, {
+        searchParams: {
+            per_page: ctx.query.limit ? parseInt(ctx.query.limit) : 100,
+        },
     });
 
-    const $ = cheerio.load(response.data);
-
-    const items = $('div.blog-posts article')
-        .slice()
-        .map((_, item) => {
-            item = $(item);
-            return {
-                title: item.find('h2 a').text(),
-                author: item.find('li.post-author span a').text(),
-                description: item.find('div.entry p').text(),
-                pubDate: parseDate(item.find('li.post-date').text()),
-                link: item.find('h2 a').attr('href'),
-            };
-        })
-        .get();
+    const items = data.map((item) => ({
+        title: item.title.rendered,
+        author: item.authors.map((a) => a.display_name).join(', '),
+        description: item.content.rendered,
+        pubDate: parseDate(item.date_gmt),
+        link: item.link,
+    }));
 
     ctx.state.data = {
         title: 'Blog - Bitmovin',
-        link: rootUrl,
+        link: `${baseUrl}/blog/`,
         item: items,
     };
 };

package/lib/v2/bjp/apod.js

@@ -0,0 +1,43 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://www.bjp.org.cn';
+    const listUrl = `${baseUrl}/APOD/list.shtml`;
+
+    const res = await got(listUrl);
+
+    const $ = cheerio.load(res.data);
+
+    const list = $('td[align=left] b')
+        .toArray()
+        .map((e) => {
+            e = $(e);
+            return {
+                title: e.find('a').attr('title'),
+                link: `${baseUrl}${e.find('a').attr('href')}`,
+                pubDate: timezone(parseDate(e.find('span').text().replace('：', ''), 'YYYY-MM-DD'), 8),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((e) =>
+            ctx.cache.tryGet(e.link, async () => {
+                const { data } = await got.get(e.link);
+                const $ = cheerio.load(data);
+
+                e.description = $('.juzhong').html();
+
+                return e;
+            })
+        )
+    );
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: '探索宇宙！每天发布一张迷人宇宙的影像，以及由专业天文学家撰写的简要说明。',
+        link: listUrl,
+        item: items,
+    };
+};

package/lib/v2/bjp/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/apod': ['HenryQW'],
+};

package/lib/v2/bjp/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'bjp.org.cn': {
+        _name: '北京天文馆',
+        '.': [
+            {
+                title: '每日一图',
+                docs: 'https://docs.rsshub.app/picture.html#bei-jing-tian-wen-guan',
+                source: ['/APOD/today.shtml', '/APOD/list.shtml', '/'],
+                target: '/bjp/apod',
+            },
+        ],
+    },
+};

package/lib/v2/bjp/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/apod', require('./apod'));
+};

package/lib/v2/bjsk/index.js

@@ -0,0 +1,52 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.bjsk.org.cn';
+
+module.exports = async (ctx) => {
+    const { path = 'newslist-1486-0-0' } = ctx.params;
+    const link = `${baseUrl}/${path}.html`;
+    const { data: response } = await got(link, {
+        https: {
+            rejectUnauthorized: false,
+        },
+    });
+    const $ = cheerio.load(response);
+
+    const list = $('.article-list a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.attr('title'),
+                link: `${baseUrl}${item.attr('href')}`,
+                pubDate: parseDate(item.find('.time').text(), 'YYYY.MM.DD'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link, {
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
+                const $ = cheerio.load(response);
+                item.description = $('.article-main').html();
+                item.author = $('.info')
+                    .text()
+                    .match(/作者：(.*)\s+来源/)[1];
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link,
+        image: 'https://www.bjsk.org.cn/favicon.ico',
+        item: items,
+    };
+};

package/lib/v2/bjsk/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:path?': ['TonyRL'],
+};

package/lib/v2/bjsk/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'bjsk.org.cn': {
+        _name: '北京社科网',
+        '.': [
+            {
+                title: '通用',
+                docs: 'https://docs.rsshub.app/government.html#bei-jing-she-ke-wang',
+                source: ['/*'],
+                target: (_, url) => `/bjsk/${url.split('/')[3].replace('.html', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/bjsk/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:path?', require('./index'));
+};

package/lib/v2/bjwxdxh/index.js

@@ -0,0 +1,52 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'http://www.bjwxdxh.org.cn';
+    const type = ctx.params.type;
+    const link = type ? `${baseUrl}/news/class/?${type}.html` : `${baseUrl}/news/class/`;
+
+    const response = await got({
+        method: 'get',
+        url: link,
+    });
+
+    const $ = cheerio.load(response.data);
+    const list = $('div#newsquery > ul > li')
+        .map((_, item) => {
+            item = $(item);
+            return {
+                title: item.find('div.title > a').text(),
+                link: new URL(item.find('div.title > a').attr('href'), baseUrl).href,
+                // pubDate: parseDate(item.find('div.time').text(), 'YYYY-MM-DD'),
+            };
+        })
+        .get();
+
+    await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(response.data);
+                const info = content('div.info')
+                    .text()
+                    .match(/作者：(.*?)\s+发布于：(.*?\s+.*?)\s/);
+                item.author = info[1];
+                item.pubDate = timezone(parseDate(info[2], 'YYYY-MM-DD HH:mm:ss'), +8);
+                item.description = content('div#con').html().trim().replace(/\n/g, '');
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link,
+        item: list,
+    };
+};

package/lib/v2/bjwxdxh/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:type?': ['Misaka13514'],
+};

package/lib/v2/bjwxdxh/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'bjwxdxh.org.cn': {
+        _name: '北京无线电协会',
+        www: [
+            {
+                title: '最新资讯',
+                docs: 'https://docs.rsshub.app/government.html#bei-jing-wu-xian-dian-xie-hui',
+                source: '/news/class/',
+                target: (params, url) => (url.includes('?') ? `/bjwxdxh/${url.split('?')[1].split('.')[0]}` : '/bjwxdxh'),
+            },
+        ],
+    },
+};

package/lib/v2/bjwxdxh/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:type?', require('./index'));
+};

package/lib/v2/blockbeats/index.js

@@ -1,50 +1,43 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const { parseRelativeDate } = require('@/utils/parse-date');
+const { parseDate } = require('@/utils/parse-date');
+const newflashApi = 'https://api.theblockbeats.info/v3/newsflash/select';
+const articleApi = 'https://api.theblockbeats.info/v3/Information/newsall';
 
 const rootUrl = 'https://www.theblockbeats.info';
 
 const channelMap = {
-    flash: '快讯',
-    news: '新闻',
+    newsflash: '快讯',
+    article: '文章',
 };
 
 module.exports = async (ctx) => {
-    const channel = ctx.params.channel ?? 'flash';
-
-    const response = await got.get(rootUrl);
-    const $ = cheerio.load(response.data);
-    const list = $('div.news-item')
-        .map((_, item) => {
-            const title = $(item).find('a').attr('title');
-            const link = rootUrl + $(item).find('a').attr('href');
-            const pubDate = parseRelativeDate($(item).find('div.article-time').text());
-
-            return {
-                title,
-                link,
-                pubDate,
-            };
-        })
-        .filter((_, item) => item.title && item.link.includes(channel))
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got.get(item.link);
-                const content = cheerio.load(detailResponse.data);
-
-                item.description = content(`div.${channel}-content`).html();
-
-                return item;
-            })
-        )
-    );
+    const channel = ctx.params.channel ?? 'newsflash';
+    const { data: response } = await got(channel === 'newsflash' ? newflashApi : articleApi);
+    const { data } = channel === 'newsflash' ? response.data : response;
+    const list = data.map((item) => ({
+        title: item.title,
+        link: `${rootUrl}/${channel === 'newsflash' ? 'flash' : 'news'}/${item.id}`,
+        description: item.content ?? item.im_abstract,
+        pubDate: parseDate(item.add_time, 'X'),
+    }));
+
+    if (channel !== 'newsflash') {
+        await Promise.all(
+            list.map((item) =>
+                ctx.cache.tryGet(item.link, async () => {
+                    const detailResponse = await got(item.link);
+                    const content = cheerio.load(detailResponse.data);
+                    item.description = content('div.news-content').html();
+                    return item;
+                })
+            )
+        );
+    }
 
     ctx.state.data = {
         title: `TheBlockBeats - ${channelMap[channel]}`,
         link: rootUrl,
-        item: items,
+        item: list,
     };
 };

package/lib/v2/blockbeats/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/:channel?': ['Fatpandac'],
+    '/:channel?': ['Fatpandac', 'jameshih'],
 };

package/lib/v2/blockbeats/radar.js

@@ -6,13 +6,13 @@ module.exports = {
                 title: '快讯',
                 docs: 'https://docs.rsshub.app/new-media.html#lu-dong-xin-wen-kuai-xun',
                 source: ['/'],
-                target: '/blockbeats/flash',
+                target: '/blockbeats/newsflash',
             },
             {
-                title: '新闻',
+                title: '文章',
                 docs: 'https://docs.rsshub.app/new-media.html#lu-dong-xin-wen-kuai-xun',
                 source: ['/'],
-                target: '/blockbeats/news',
+                target: '/blockbeats/article',
             },
         ],
     },

package/lib/v2/bloomberg/authors.js

@@ -0,0 +1,52 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const rssParser = require('@/utils/rss-parser');
+const { asyncPoolAll, parseArticle } = require('./utils');
+
+const parseAuthorNewsList = async (slug) => {
+    const baseURL = `https://www.bloomberg.com/authors/${slug}`;
+    const apiUrl = `https://www.bloomberg.com/lineup/api/lazy_load_author_stories?slug=${slug}&authorType=default&page=1`;
+    const resp = await got(apiUrl);
+    // Likely rate limited
+    if (!resp.data.html) {
+        return [];
+    }
+    const $ = cheerio.load(resp.data.html);
+    const articles = $('article.story-list-story');
+    return articles
+        .map((index, item) => {
+            item = $(item);
+            const headline = item.find('a.story-list-story__info__headline-link');
+            return {
+                title: headline.text(),
+                pubDate: item.attr('data-updated-at'),
+                guid: `bloomberg:${item.attr('data-id')}`,
+                link: new URL(headline.attr('href'), baseURL).href,
+            };
+        })
+        .get();
+};
+
+module.exports = async (ctx) => {
+    const { id, slug, source } = ctx.params;
+    const link = `https://www.bloomberg.com/authors/${id}/${slug}`;
+
+    let list = [];
+    if (!source || source === 'api') {
+        list = await parseAuthorNewsList(`${id}/${slug}`);
+    }
+    // Fallback to rss if api failed or requested by param
+    if (source === 'rss' || list.length === 0) {
+        list = (await rssParser.parseURL(`${link}.rss`)).items;
+    }
+
+    const item = await asyncPoolAll(1, list, (item) => parseArticle(item, ctx));
+    const authorName = item.find((i) => i.author)?.author ?? 'Unknown';
+
+    ctx.state.data = {
+        title: `Bloomberg - ${authorName}`,
+        link,
+        language: 'en-us',
+        item,
+    };
+};

package/lib/v2/bloomberg/index.js

@@ -1,5 +1,4 @@
-const { rootUrl, parseNewsList, parseArticle } = require('./utils');
-const asyncPool = require('tiny-async-pool');
+const { rootUrl, asyncPoolAll, parseNewsList, parseArticle } = require('./utils');
 const site_title_mapping = {
     '/': 'News',
     bpol: 'Politics',
@@ -15,14 +14,6 @@ const site_title_mapping = {
     citylab: 'CityLab',
 };
 
-const asyncPoolAll = async (...args) => {
-    const results = [];
-    for await (const result of asyncPool(...args)) {
-        results.push(result);
-    }
-    return results;
-};
-
 module.exports = async (ctx) => {
     const { site } = ctx.params;
     const currentUrl = site ? `${rootUrl}/${site}/sitemap_news.xml` : `${rootUrl}/sitemap_news.xml`;

package/lib/v2/bloomberg/maintainer.js

@@ -1,4 +1,5 @@
 module.exports = {
+    '/authors/:id/:slug/:source?': ['josh'],
     '/:site?': ['bigfei'],
     '/': ['bigfei'],
 };