rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/lib/routes/mastodon/timeline_remote.js

@@ -1,9 +1,13 @@
 const got = require('@/utils/got');
 const utils = require('./utils');
+const config = require('@/config').value;
 
 module.exports = async (ctx) => {
     const site = ctx.params.site;
     const only_media = ctx.params.only_media ? 'true' : 'false';
+    if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) {
+        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+    }
 
     const url = `http://${site}/api/v1/timelines/public?remote=true&only_media=${only_media}`;
 

package/lib/routes/mastodon/utils.js

@@ -1,6 +1,8 @@
 const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
 
+const allowSiteList = ['mastodon.social', 'pawoo.net'];
+
 const parseStatuses = (data) =>
     data.map((item) => {
         // docs on: https://docs.joinmastodon.org/entities/status/
@@ -125,4 +127,5 @@ module.exports = {
     parseStatuses,
     getAccountStatuses,
     getAccountIdByAcct,
+    allowSiteList,
 };

package/lib/routes/pornhub/category_url.js

@@ -1,10 +1,14 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const language = ctx.params.language || 'www';
     const url = ctx.params.url || 'video';
     const link = `https://${language}.pornhub.com/${url}`;
+    if (!isValidHost(language)) {
+        throw Error('Invalid language');
+    }
 
     const response = await got.get(link);
     const $ = cheerio.load(response.data);

package/lib/routes/pornhub/model.js

@@ -1,11 +1,15 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const language = ctx.params.language || 'www';
     const username = ctx.params.username;
     const sort = ctx.params.sort || 'mr';
     const link = `https://${language}.pornhub.com/model/${username}/videos?o=${sort}`;
+    if (!isValidHost(language)) {
+        throw Error('Invalid language');
+    }
 
     const response = await got.get(link);
     const $ = cheerio.load(response.data);

package/lib/routes/pornhub/pornstar.js

@@ -1,11 +1,15 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const language = ctx.params.language || 'www';
     const username = ctx.params.username;
     const sort = ctx.params.sort || 'mr';
     const link = `https://${language}.pornhub.com/pornstar/${username}/videos?o=${sort}`;
+    if (!isValidHost(language)) {
+        throw Error('Invalid language');
+    }
 
     const response = await got.get(link);
     const $ = cheerio.load(response.data);

package/lib/routes/pornhub/users.js

@@ -1,10 +1,14 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const language = ctx.params.language || 'www';
     const username = ctx.params.username;
     const link = `https://${language}.pornhub.com/users/${username}/videos`;
+    if (!isValidHost(language)) {
+        throw Error('Invalid language');
+    }
 
     const response = await got.get(link);
     const $ = cheerio.load(response.data);

package/lib/routes/touhougarakuta/index.js

@@ -11,6 +11,9 @@ const getBaseUrl = (language) => (language === 'ja' ? 'https://touhougarakuta.co
 
 module.exports = async (ctx) => {
     const { language, type } = ctx.params;
+    if (!Object.keys(languageCodes).includes(language)) {
+        throw Error('Invalid language');
+    }
 
     const baseUrl = getBaseUrl(language);
 

package/lib/routes/weforum/report.js

@@ -1,13 +1,17 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
-    ctx.params.lang = ctx.params.lang || 'www';
-    ctx.params.platform = ctx.params.platform || '';
-    ctx.params.year = ctx.params.year || '';
-
-    const rootUrl = `https://${ctx.params.lang === 'en' ? 'www' : ctx.params.lang}.weforum.org`;
-    const currentUrl = `${rootUrl}/reports?platform=${ctx.params.platform}&year=${ctx.params.year}`;
+    const lang = ctx.params.lang || 'www';
+    const platform = ctx.params.platform || '';
+    const year = ctx.params.year || '';
+    if (!isValidHost(lang)) {
+        throw Error('Invalid lang');
+    }
+
+    const rootUrl = `https://${lang === 'en' ? 'www' : lang}.weforum.org`;
+    const currentUrl = `${rootUrl}/reports?platform=${platform}&year=${year}`;
     const response = await got({
         method: 'get',
         url: currentUrl,

package/lib/routes/ziroom/room.js

@@ -1,4 +1,5 @@
 const got = require('@/utils/got');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const city = ctx.params.city || 'sh';
@@ -7,6 +8,10 @@ module.exports = async (ctx) => {
     const room = ctx.params.room || '1';
     const domain = `${city === 'bj' ? '' : city + '.'}m.ziroom.com`;
 
+    if (!isValidHost(city)) {
+        throw Error('Invalid city');
+    }
+
     const response = await got({
         method: 'post',
         url: `http://${domain}/list/ajax-get-data`,

package/lib/utils/render.js

@@ -2,6 +2,52 @@ const art = require('art-template');
 
 // We may add more control over it later
 
+/**
+ * This function should be used by RSSHub middleware only.
+ * @param {object} data ctx.state.data
+ * @returns `JSON.stringify`-ed [JSON Feed](https://www.jsonfeed.org/)
+ */
+const json = (data) => {
+    const jsonFeed = {
+        version: 'https://jsonfeed.org/version/1.1',
+        title: data.title || 'RSSHub',
+        home_page_url: data.link || 'https://docs.rsshub.app',
+        feed_url: data.feedLink,
+        description: `${data.description || data.title} - Made with love by RSSHub(https://github.com/DIYgod/RSSHub)`,
+        icon: data.image,
+        authors: typeof data.author === 'string' ? [{ name: data.author }] : data.author,
+        language: data.language || 'zh-cn',
+        items: data.item.map((item) => ({
+            id: item.guid || item.id || item.link,
+            url: item.link,
+            title: item.title,
+            content_html: (item.content && item.content.html) || item.description || item.title,
+            content_text: item.content && item.content.text,
+            image: item.image,
+            banner_image: item.banner,
+            date_published: item.pubDate,
+            date_modified: item.updated,
+            authors: typeof item.author === 'string' ? [{ name: item.author }] : item.author,
+            tags: typeof item.category === 'string' ? [item.category] : item.category,
+            language: item.language,
+            attachments: item.enclosure_url
+                ? [
+                      {
+                          url: item.enclosure_url,
+                          mime_type: item.enclosure_type,
+                          title: item.enclosure_title,
+                          size_in_bytes: item.enclosure_length,
+                          duration_in_seconds: item.itunes_duration,
+                      },
+                  ]
+                : undefined,
+            _extra: item._extra || undefined,
+        })),
+    };
+    return JSON.stringify(jsonFeed, null, 4);
+};
+
 module.exports = {
     art,
+    json,
 };

package/lib/utils/valid-host.js

@@ -0,0 +1,16 @@
+/**
+ * Check if a sub-domain is valid
+ * @param {String} hostname sub-domain
+ * @returns {Boolean} true if valid
+ */
+const isValidHost = (hostname) => {
+    if (typeof hostname !== 'string') {
+        return false;
+    }
+    const regex = /^[a-zA-Z0-9]([a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?$/;
+    return regex.test(hostname);
+};
+
+module.exports = {
+    isValidHost,
+};

package/lib/v2/18comic/album.js

@@ -38,13 +38,14 @@ module.exports = async (ctx) => {
             return {
                 title: item.text(),
                 link: `${rootUrl}${item.attr('href')}`,
+                guid: `https://18comic.org${item.attr('href')}`,
                 pubDate: parseDate(item.find('.hidden-xs').text()),
             };
         });
 
     items = await Promise.all(
         items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
+            ctx.cache.tryGet(item.guid, async () => {
                 const detailResponse = await got({
                     method: 'get',
                     url: item.link,

package/lib/v2/18comic/blogs.js

@@ -24,12 +24,13 @@ module.exports = async (ctx) => {
             return {
                 title: item.text(),
                 link: `${rootUrl}${item.parent().attr('href')}`,
+                guid: `https://18comic.org${item.parent().attr('href')}`,
             };
         });
 
     items = await Promise.all(
         items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
+            ctx.cache.tryGet(item.guid, async () => {
                 const detailResponse = await got({
                     method: 'get',
                     url: item.link,

package/lib/v2/18comic/radar.js

@@ -1,31 +1,41 @@
+const _18comic = {
+    _name: '禁漫天堂',
+    '.': [
+        {
+            title: '成人 A 漫',
+            docs: 'https://docs.rsshub.app/anime.html#jin-man-tian-tang-cheng-ren-a-man',
+            source: ['/'],
+            target: '/18comic/:category?/:time?/:order?/:keyword?',
+        },
+        {
+            title: '搜索',
+            docs: 'https://docs.rsshub.app/anime.html#jin-man-tian-tang-sou-suo',
+            source: ['/'],
+            target: '/18comic/search/:option?/:category?:keyword?/:time?/:order?',
+        },
+        {
+            title: '专辑',
+            docs: 'https://docs.rsshub.app/anime.html#jin-man-tian-tang-zhuan-ji',
+            source: ['/'],
+            target: '/18comic/album/:id',
+        },
+        {
+            title: '文庫',
+            docs: 'https://docs.rsshub.app/anime.html#jin-man-tian-tang-wen-ku',
+            source: ['/'],
+            target: '/18comic/blogs/:category?',
+        },
+    ],
+};
+
 module.exports = {
-    '18comic.org': {
-        _name: '18comic 禁漫天堂',
-        '.': [
-            {
-                title: '成人 A 漫',
-                docs: 'https://docs.rsshub.app/anime.html#18comic-jin-man-tian-tang-cheng-ren-a-man',
-                source: ['/'],
-                target: '/18comic/:category?/:time?/:order?/:keyword?',
-            },
-            {
-                title: '搜索',
-                docs: 'https://docs.rsshub.app/anime.html#18comic-jin-man-tian-tang-sou-suo',
-                source: ['/'],
-                target: '/18comic/search/:option?/:category?:keyword?/:time?/:order?',
-            },
-            {
-                title: '专辑',
-                docs: 'https://docs.rsshub.app/anime.html#18comic-jin-man-tian-tang-zhuan-ji',
-                source: ['/'],
-                target: '/18comic/album/:id',
-            },
-            {
-                title: '文庫',
-                docs: 'https://docs.rsshub.app/anime.html#18comic-jin-man-tian-tang-wen-ku',
-                source: ['/'],
-                target: '/18comic/blogs/:category?',
-            },
-        ],
-    },
+    '18comic.org': _18comic,
+    '18comic.vip': _18comic,
+    //
+    'jmcomic.me': _18comic,
+    'jmcomic1.me': _18comic,
+    //
+    'jmcomic.rocks': _18comic,
+    'jmcomic1.rocks': _18comic,
+    'jmcomic.group': _18comic,
 };

package/lib/v2/18comic/utils.js

@@ -2,17 +2,15 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 
-const rootUrl = 'https://18comic.org';
+const rootUrl = 'https://jmcomic.me';
+// list of address: https://jmcomic1.bet
 
 module.exports = {
     rootUrl,
     ProcessItems: async (ctx, currentUrl) => {
         currentUrl = currentUrl.replace(/\?$/, '');
 
-        const response = await got({
-            method: 'get',
-            url: currentUrl,
-        });
+        const response = await got(currentUrl);
 
         const $ = cheerio.load(response.data);
 
@@ -25,16 +23,14 @@ module.exports = {
                 return {
                     title: item.text().trim(),
                     link: `${rootUrl}${item.prev().find('a').attr('href')}`,
+                    guid: `https://18comic.org${item.prev().find('a').attr('href')}`,
                 };
             });
 
         items = await Promise.all(
             items.map((item) =>
-                ctx.cache.tryGet(item.link, async () => {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                    });
+                ctx.cache.tryGet(item.guid, async () => {
+                    const detailResponse = await got(item.link);
 
                     const content = cheerio.load(detailResponse.data);
 

package/lib/v2/19lou/index.js

@@ -3,6 +3,7 @@ const cheerio = require('cheerio');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const iconv = require('iconv-lite');
+const { isValidHost } = require('@/utils/valid-host');
 
 const setCookie = function (cookieName, cookieValue, seconds, path, domain, secure) {
     let expires = null;
@@ -15,6 +16,9 @@ const setCookie = function (cookieName, cookieValue, seconds, path, domain, secu
 
 module.exports = async (ctx) => {
     const city = ctx.params.city ?? 'www';
+    if (!isValidHost(city)) {
+        throw Error('Invalid city');
+    }
 
     const rootUrl = `https://${city}.19lou.com`;
 

package/lib/v2/91porn/author.js

@@ -3,11 +3,13 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { domainValidation } = require('./utils');
 
 module.exports = async (ctx) => {
     const { domain = '91porn.com' } = ctx.query;
     const { uid, lang = 'en_US' } = ctx.params;
     const siteUrl = `https://${domain}/uvideos.php?UID=${uid}&type=public`;
+    domainValidation(domain, ctx);
 
     const response = await got.post(siteUrl, {
         form: {

package/lib/v2/91porn/index.js

@@ -3,11 +3,13 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { domainValidation } = require('./utils');
 
 module.exports = async (ctx) => {
     const { domain = '91porn.com' } = ctx.query;
     const siteUrl = `https://${domain}/index.php`;
     const { lang = 'en_US' } = ctx.params;
+    domainValidation(domain, ctx);
 
     const response = await got.post(siteUrl, {
         form: {

package/lib/v2/91porn/utils.js

@@ -0,0 +1,12 @@
+const config = require('@/config').value;
+const allowDomain = ['91porn.com', 'www.91porn.com', '0122.91p30.com', 'www.91zuixindizhi.com', 'w1218.91p46.com'];
+
+const domainValidation = (domain, ctx) => {
+    if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(domain)) {
+        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
+    }
+};
+
+module.exports = {
+    domainValidation,
+};

package/lib/v2/baai/comments.js

@@ -0,0 +1,35 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const { baseUrl, apiHost } = require('./utils');
+
+module.exports = async (ctx) => {
+    const responses = await got.all(
+        Array.from(
+            {
+                // first 2 pages
+                length: (ctx.query.limit ? parseInt(ctx.query.limit, 10) : 12) / 6,
+            },
+            (_, v) => `${apiHost}/api/v1/commentsall?page=${v + 1}`
+        ).map((url) => got.post(url))
+    );
+
+    const items = responses
+        .map((response) => response.data.data.list)
+        .flat()
+        .map((item) => ({
+            title: `${item.user_info.name} ${item.title_desc} ${item.title}`,
+            description: item.desc,
+            pubDate: timezone(parseDate(item.created_at), +8),
+            author: item.user_info.name,
+            link: `${baseUrl}/view/${item.source_id || item.story_id}`,
+            guid: `baai:hub:comments:${item.id}`,
+        }));
+
+    ctx.state.data = {
+        title: '评论 - 智源社区',
+        link: `${baseUrl}/comments`,
+        item: items,
+    };
+};

package/lib/v2/baai/events.js

@@ -0,0 +1,35 @@
+const got = require('@/utils/got');
+
+const { baseUrl, apiHost, parseEventDetail, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const responses = await got.all(
+        Array.from(
+            {
+                // first 2 pages
+                length: (ctx.query.limit ? parseInt(ctx.query.limit, 10) : 16) / 8,
+            },
+            (_, v) => `${apiHost}/api/v1/events?page=${v + 1}`
+        ).map((url) => got.post(url))
+    );
+
+    const list = responses
+        .map((response) => response.data.data)
+        .flat()
+        .map((item) => parseItem(item));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                item.description = await parseEventDetail(item);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '活动 - 智源社区',
+        link: `${baseUrl}/events`,
+        item: items,
+    };
+};

package/lib/v2/baai/hub.js

@@ -0,0 +1,60 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const { baseUrl, apiHost, parseEventDetail, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { tagId, sort = 'new', range } = ctx.params;
+
+    let title, description, brief, iconUrl;
+    if (tagId) {
+        const { data } = await got(`${apiHost}/api/v1/tags`, {
+            searchParams: {
+                id: tagId,
+            },
+        });
+        title = data.data[0].title;
+        description = data.data[0].description;
+        brief = data.data[0].brief;
+        iconUrl = data.data[0].icon_url;
+    }
+
+    const responses = await got.all(
+        Array.from(
+            {
+                // first 3 pages
+                length: (ctx.query.limit ? parseInt(ctx.query.limit, 10) : 18) / 6,
+            },
+            (_, v) => `${apiHost}/api/v1/story/list?page=${v + 1}&sort=${sort}${tagId ? `&tag_id=${tagId}` : ''}${range ? `&time_range=${range}` : ''}`
+        ).map((url) => got.post(url))
+    );
+
+    const list = responses
+        .filter((response) => response.data.data)
+        .map((response) => response.data.data)
+        .flat()
+        .map((item) => parseItem(item));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (!item.eventId) {
+                    const { data: response } = await got(item.link);
+                    const $ = cheerio.load(response);
+                    item.description = item.is_event ? $('div.box2').html() : $('.post-content').html();
+                } else {
+                    item.description = await parseEventDetail(item);
+                }
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${title ? `${title} - ` : ''}${description ? `${description} - ` : ''}智源社区`,
+        description: brief,
+        link: `${baseUrl}/?${tagId ? `tag_id=${tagId}&` : ''}sort=${sort}${range ? `&time_range=${range}` : ''}`,
+        image: iconUrl,
+        item: items,
+    };
+};

package/lib/v2/baai/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/hub/comments': ['TonyRL'],
+    '/hub/events': ['TonyRL'],
+    '/hub/:tagId?/:sort?/:range?': ['TonyRL'],
+};

package/lib/v2/baai/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'baai.ac.cn': {
+        _name: '北京智源人工智能研究院',
+        hub: [
+            {
+                title: '智源社区',
+                docs: 'https://docs.rsshub.app/programming.html#bei-jing-zhi-yuan-ren-gong-zhi-neng-yan-jiu-yuan',
+                source: ['/'],
+                target: (params, url) => {
+                    const searchParams = new URL(url).searchParams;
+                    const tagId = searchParams.get('tag_id');
+                    const sort = searchParams.get('sort');
+                    const range = searchParams.get('time_range');
+                    return `/baai/hub${tagId ? `/${tagId}` : ''}${sort ? `/${sort}` : ''}${range ? `/${range}` : ''}`;
+                },
+            },
+            {
+                title: '活动 - 智源社区',
+                docs: 'https://docs.rsshub.app/programming.html#bei-jing-zhi-yuan-ren-gong-zhi-neng-yan-jiu-yuan',
+                source: ['/events', '/'],
+                target: '/baai/hub/events',
+            },
+            {
+                title: '评论 - 智源社区',
+                docs: 'https://docs.rsshub.app/programming.html#bei-jing-zhi-yuan-ren-gong-zhi-neng-yan-jiu-yuan',
+                source: ['/comments', '/'],
+                target: '/baai/hub/comments',
+            },
+        ],
+    },
+};

package/lib/v2/baai/router.js

@@ -0,0 +1,7 @@
+module.exports = (router) => {
+    router.get('/hub/comments', require('./comments'));
+    router.get('/hub/events', require('./events'));
+    router.get('/hub/:tagId/:sort?/:range?', require('./hub'));
+    router.get('/hub/:tagId/:sort?', require('./hub'));
+    router.get('/hub/:sort?', require('./hub'));
+};

package/lib/v2/baai/utils.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const baseUrl = 'https://hub.baai.ac.cn';
+const eventUrl = 'https://event.baai.ac.cn';
+const apiHost = 'https://hub-api.baai.ac.cn';
+
+const parseItem = (item) => ({
+    link: item.is_event ? `${eventUrl}/activities/${item.event_info.id}` : `${baseUrl}/view/${item.story_id}`,
+    title: item.is_event ? item.event_info.name : item.story_info.title,
+    pubDate: timezone(parseDate(item.is_event ? item.event_info.time_desc : item.story_info.created_at.replace('发布', '').replace('分享', '')), 8),
+    author: item.is_event ? item.event_info.company : item.story_info.user_name,
+    category: item.is_event ? null : item.story_info.tag_names.map((tag) => tag.title),
+    eventId: item.is_event ? item.event_info.id : null,
+});
+
+const parseEventDetail = async (item) => {
+    const { data } = await got(`${eventUrl}/api/api/Activity/IntroductionTypes`, {
+        searchParams: {
+            activityId: item.eventId,
+        },
+    });
+    return data.data.ac_desc + data.data.ac_desc_two;
+};
+
+module.exports = {
+    baseUrl,
+    eventUrl,
+    apiHost,
+    parseItem,
+    parseEventDetail,
+};

package/lib/v2/baidu/maintainer.js

@@ -5,4 +5,5 @@ module.exports = {
     '/tieba/post/:id': ['u3u'],
     '/tieba/post/lz/:id': ['u3u'],
     '/tieba/user/:uid': ['igxlin', 'nczitzk'],
+    '/top/:board?': ['xyqfer'],
 };