rsshub 1.0.0-master.fdbb6fa → 1.0.0-master.fdc59d2

package/lib/config.js

@@ -2,6 +2,8 @@ require('dotenv').config();
 const randUserAgent = require('./utils/rand-user-agent');
 let envs = process.env;
 let value;
+const TRUE_UA = 'RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)';
+
 const calculateValue = () => {
     const bilibili_cookies = {};
     const twitter_tokens = {};
@@ -40,7 +42,8 @@ const calculateValue = () => {
         listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1
         requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数
         requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request
-        ua: envs.UA || randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }),
+        ua: envs.UA ? envs.UA : envs.NO_RANDOM_UA === 'true' || envs.NO_RANDOM_UA === '1' ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }),
+        trueUA: TRUE_UA,
         // cors request
         allowOrigin: envs.ALLOW_ORIGIN,
         // cache
@@ -134,6 +137,7 @@ const calculateValue = () => {
             ipb_pass_hash: envs.EH_IPB_PASS_HASH,
             sk: envs.EH_SK,
             igneous: envs.EH_IGNEOUS,
+            star: envs.EH_STAR,
             img_proxy: envs.EH_IMG_PROXY,
         },
         email: {
@@ -148,6 +152,9 @@ const calculateValue = () => {
             username: envs.FANFOU_USERNAME,
             password: envs.FANFOU_PASSWORD,
         },
+        fantia: {
+            cookies: envs.FANTIA_COOKIE,
+        },
         game4399: {
             cookie: envs.GAME_4399,
         },
@@ -178,9 +185,15 @@ const calculateValue = () => {
             password: envs.IG_PASSWORD,
             proxy: envs.IG_PROXY,
         },
+        iwara: {
+            cookie: envs.IWARA_COOKIE,
+        },
         lastfm: {
             api_key: envs.LASTFM_API_KEY,
         },
+        manhuagui: {
+            cookie: envs.MHGUI_COOKIE,
+        },
         mastodon: {
             apiHost: envs.MASTODON_API_HOST,
             accessToken: envs.MASTODON_API_ACCESS_TOKEN,
@@ -217,6 +230,9 @@ const calculateValue = () => {
         pkubbs: {
             cookie: envs.PKUBBS_COOKIE,
         },
+        saraba1st: {
+            cookie: envs.SARABA1ST_COOKIE,
+        },
         scboy: {
             token: envs.SCBOY_BBS_TOKEN,
         },
@@ -235,8 +251,8 @@ const calculateValue = () => {
             consumer_key: envs.TWITTER_CONSUMER_KEY,
             consumer_secret: envs.TWITTER_CONSUMER_SECRET,
             tokens: twitter_tokens,
-            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
-            // 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
+            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw',
+            // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25
         },
         weibo: {
             app_key: envs.WEIBO_APP_KEY,
@@ -268,6 +284,9 @@ const calculateValue = () => {
         zhihu: {
             cookies: envs.ZHIHU_COOKIES,
         },
+        zodgame: {
+            cookie: envs.ZODGAME_COOKIE,
+        },
     };
 };
 calculateValue();

package/lib/middleware/cache/index.js

@@ -56,6 +56,9 @@ module.exports = function (app) {
     app.context.cache = {
         ...cacheModule,
         tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire, refresh = true) => {
+            if (typeof key !== 'string') {
+                throw Error('Cache key must be a string');
+            }
             let v = await get(key, refresh);
             if (!v) {
                 v = await getValueFunc();

package/lib/middleware/parameter.js

@@ -54,7 +54,9 @@ module.exports = async (ctx, next) => {
         ctx.state.data.description && (ctx.state.data.description = entities.decodeXML(ctx.state.data.description + ''));
 
         // sort items
-        ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
+        if (ctx.query.sorted !== 'false') {
+            ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
+        }
 
         const handleItem = (item) => {
             item.title && (item.title = entities.decodeXML(item.title + ''));
@@ -250,12 +252,10 @@ module.exports = async (ctx, next) => {
                     const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
                         // if parser failed, return default description and not report error
                         try {
-                            mercury_parser = mercury_parser || require('@postlight/mercury-parser');
+                            mercury_parser = mercury_parser || require('@postlight/parser');
 
-                            const res = await got(link);
-                            const $ = cheerio.load(res.data, {
-                                xmlMode: true,
-                            });
+                            const { data: res } = await got(link);
+                            const $ = cheerio.load(res);
                             const result = await mercury_parser.parse(link, {
                                 html: $.html(),
                             });
@@ -266,7 +266,7 @@ module.exports = async (ctx, next) => {
                     });
 
                     item.author = author || (parsed_result ? parsed_result.author : '');
-                    item.description = parsed_result ? parsed_result.content : description;
+                    item.description = parsed_result ? entities.decodeXML(parsed_result.content) : description;
                 });
                 await Promise.all(tasks);
             }

package/lib/radar-rules.js

@@ -45,41 +45,6 @@ module.exports = {
             },
         ],
     },
-    'pixiv.net': {
-        _name: 'Pixiv',
-        www: [
-            {
-                title: '用户收藏',
-                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
-                source: '/users/:id/bookmarks/artworks',
-                target: '/pixiv/user/bookmarks/:id',
-            },
-            {
-                title: '用户动态',
-                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
-                source: '/users/:id',
-                target: '/pixiv/user/:id',
-            },
-            {
-                title: '排行榜',
-                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
-                source: '/ranking.php',
-                target: (params, url) => `/pixiv/ranking/${new URL(url).searchParams.get('mode') || 'daily'}`,
-            },
-            {
-                title: '关键词',
-                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
-                source: ['/tags/:keyword', '/tags/:keyword/:type?'],
-                target: (params, url) => `/pixiv/search/:keyword/${new URL(url).searchParams.get('order')}/${new URL(url).searchParams.get('mode')}`,
-            },
-            {
-                title: '关注的新作品',
-                docs: 'https://docs.rsshub.app/social-media.html#pixiv',
-                source: '/bookmark_new_illust.php',
-                target: '/pixiv/user/illustfollows',
-            },
-        ],
-    },
     'smzdm.com': {
         _name: '什么值得买',
         www: [
@@ -202,39 +167,6 @@ module.exports = {
             },
         ],
     },
-    'manhuagui.com': {
-        _name: '漫画柜',
-        www: [
-            {
-                title: '漫画更新',
-                docs: 'https://docs.rsshub.app/anime.html#kan-man-hua',
-                source: '/comic/:id/',
-                target: '/manhuagui/comic/:id',
-            },
-        ],
-    },
-    'mhgui.com': {
-        _name: '漫画柜镜像站',
-        www: [
-            {
-                title: '漫画更新',
-                docs: 'https://docs.rsshub.app/anime.html#kan-man-hua-jing-xiang-zhan',
-                source: '/comic/:id/',
-                target: '/mhgui/comic/:id',
-            },
-        ],
-    },
-    'tw.manhuagui.com': {
-        _name: '漫画柜台湾',
-        www: [
-            {
-                title: '漫画更新',
-                docs: 'https://docs.rsshub.app/anime.html#kan-man-hua-tai-wan',
-                source: '/comic/:id/',
-                target: '/twmanhuagui/comic/:id',
-            },
-        ],
-    },
     'pgyer.com': {
         _name: '蒲公英应用分发',
         www: [
@@ -246,56 +178,6 @@ module.exports = {
             },
         ],
     },
-    'baidu.com': {
-        _name: '贴吧',
-        tieba: [
-            {
-                title: '帖子列表',
-                docs: 'https://docs.rsshub.app/bbs.html#tie-ba',
-                source: 'f',
-                target: (params, url) => {
-                    const type = new URL(url).searchParams.get('tab');
-                    if (!type || type === 'main') {
-                        return `/tieba/forum/${new URL(url).searchParams.get('kw')}`;
-                    }
-                },
-            },
-            {
-                title: '精品帖子',
-                docs: 'https://docs.rsshub.app/bbs.html#tie-ba',
-                source: 'f',
-                target: (params, url) => {
-                    const type = new URL(url).searchParams.get('tab');
-                    if (type === 'good') {
-                        return `/tieba/forum/good/${new URL(url).searchParams.get('kw')}`;
-                    }
-                },
-            },
-            {
-                title: '帖子动态',
-                docs: 'https://docs.rsshub.app/bbs.html#tie-ba',
-                source: '/p/:id',
-                target: '/tieba/post/:id',
-            },
-            {
-                title: '只看楼主',
-                docs: 'https://docs.rsshub.app/bbs.html#tie-ba',
-                source: '/p/:id',
-                target: '/tieba/post/lz/:id',
-            },
-            {
-                title: '用户帖子',
-                docs: 'https://docs.rsshub.app/bbs.html#tie-ba',
-                source: '/home/main',
-                target: (params, url) => {
-                    const uid = new URL(url).searchParams.get('un');
-                    if (uid) {
-                        return `/tieba/user/${uid}`;
-                    }
-                },
-            },
-        ],
-    },
     'wineyun.com': {
         _name: '酒云网',
         www: [
@@ -396,35 +278,6 @@ module.exports = {
             },
         ],
     },
-    'dcard.tw': {
-        _name: 'Dcard',
-        www: [
-            {
-                title: '首頁帖子-最新',
-                docs: 'https://docs.rsshub.app/bbs.html#dcard',
-                source: '/f',
-                target: '/dcard/posts/latest',
-            },
-            {
-                title: '首頁帖子-熱門',
-                docs: 'https://docs.rsshub.app/bbs.html#dcard',
-                source: '/f',
-                target: '/dcard/posts/popular',
-            },
-            {
-                title: '板塊帖子-最新',
-                docs: 'https://docs.rsshub.app/bbs.html#dcard',
-                source: '/f/:section',
-                target: '/dcard/:section/latest',
-            },
-            {
-                title: '板塊帖子-熱門',
-                docs: 'https://docs.rsshub.app/bbs.html#dcard',
-                source: '/f/:section',
-                target: '/dcard/:section/popular',
-            },
-        ],
-    },
     'wegene.com': {
         _name: 'WeGene',
         www: [
@@ -832,37 +685,6 @@ module.exports = {
             },
         ],
     },
-    'qidian.com': {
-        _name: '起点',
-        book: [
-            {
-                title: '章节',
-                docs: 'https://docs.rsshub.app/reading.html#qi-dian',
-                source: '/info/:id',
-                target: '/qidian/chapter/:id',
-            },
-            {
-                title: '讨论区',
-                docs: 'https://docs.rsshub.app/reading.html#qi-dian',
-                source: '/info/:id',
-                target: '/qidian/forum/:id',
-            },
-        ],
-        www: [
-            {
-                title: '限免',
-                docs: 'https://docs.rsshub.app/reading.html#qi-dian',
-                source: '/free',
-                target: '/qidian/free',
-            },
-            {
-                title: '女生限免',
-                docs: 'https://docs.rsshub.app/reading.html#qi-dian',
-                source: '/mm/free',
-                target: '/qidian/free/mm',
-            },
-        ],
-    },
     'hackerone.com': {
         _name: 'HackerOne',
         '.': [
@@ -1199,59 +1021,59 @@ module.exports = {
             },
         ],
     },
-    'zcool.com.cn': {
-        _name: '站酷',
-        www: [
-            {
-                title: '发现',
-                docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
-                target: '/zcool/discover',
-            },
-            {
-                title: '发现 - 精选 - 全部推荐',
-                docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
-                target: '/zcool/discover/all',
-            },
-            {
-                title: '发现 - 精选 - 首页推荐',
-                docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
-                target: '/zcool/discover/home',
-            },
-            {
-                title: '发现 - 精选 - 编辑精选',
-                docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
-                target: '/zcool/discover/home',
-            },
-            {
-                title: '发现 - 精选 - 文章 - 编辑精选',
-                docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
-                target: '/zcool/discover/article',
-            },
-            {
-                title: '作品榜单',
-                docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
-                target: '/zcool/top/design',
-            },
-            {
-                title: '文章榜单',
-                docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
-                target: '/zcool/top/article',
-            },
-            {
-                title: '用户作品',
-                docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: ['/u/:id'],
-                target: `/zcool/user/:id`,
-            },
-        ],
-    },
+    // 'zcool.com.cn': {
+    //     _name: '站酷',
+    //     www: [
+    //         {
+    //             title: '发现',
+    //             docs: 'https://docs.rsshub.app/design.html#zhan-ku',
+    //             source: '',
+    //             target: '/zcool/discover',
+    //         },
+    //         {
+    //             title: '发现 - 精选 - 全部推荐',
+    //             docs: 'https://docs.rsshub.app/design.html#zhan-ku',
+    //             source: '',
+    //             target: '/zcool/discover/all',
+    //         },
+    //         {
+    //             title: '发现 - 精选 - 首页推荐',
+    //             docs: 'https://docs.rsshub.app/design.html#zhan-ku',
+    //             source: '',
+    //             target: '/zcool/discover/home',
+    //         },
+    //         {
+    //             title: '发现 - 精选 - 编辑精选',
+    //             docs: 'https://docs.rsshub.app/design.html#zhan-ku',
+    //             source: '',
+    //             target: '/zcool/discover/home',
+    //         },
+    //         {
+    //             title: '发现 - 精选 - 文章 - 编辑精选',
+    //             docs: 'https://docs.rsshub.app/design.html#zhan-ku',
+    //             source: '',
+    //             target: '/zcool/discover/article',
+    //         },
+    //         {
+    //             title: '作品榜单',
+    //             docs: 'https://docs.rsshub.app/design.html#zhan-ku',
+    //             source: '',
+    //             target: '/zcool/top/design',
+    //         },
+    //         {
+    //             title: '文章榜单',
+    //             docs: 'https://docs.rsshub.app/design.html#zhan-ku',
+    //             source: '',
+    //             target: '/zcool/top/article',
+    //         },
+    //         {
+    //             title: '用户作品',
+    //             docs: 'https://docs.rsshub.app/design.html#zhan-ku',
+    //             source: ['/u/:id'],
+    //             target: `/zcool/user/:id`,
+    //         },
+    //     ],
+    // },
     'zhuixinfan.com': {
         _name: '追新番日剧站',
         '.': [
@@ -1694,20 +1516,6 @@ module.exports = {
             },
         ],
     },
-    'saraba1st.com': {
-        _name: 'Saraba1st',
-        bbs: [
-            {
-                title: '帖子',
-                docs: 'https://docs.rsshub.app/bbs.html#saraba1st',
-                source: '/2b/:id',
-                target: (params) => {
-                    const id = params.id.includes('thread') ? params.id.split('-')[1] : '';
-                    return id ? `/saraba1st/thread/${id}` : '';
-                },
-            },
-        ],
-    },
     'scboy.com': {
         _name: 'scboy 论坛',
         www: [
@@ -1850,65 +1658,6 @@ module.exports = {
             },
         ],
     },
-    'bgm.tv': {
-        _name: 'Bangumi',
-        '.': [
-            {
-                title: '小组话题',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/group/:id',
-                target: '/bangumi/group/:id',
-            },
-            {
-                title: '小组话题的新回复',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/group/topic/:id',
-                target: '/bangumi/topic/:id',
-            },
-            {
-                title: '现实人物的新作品',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/person/:id',
-                target: '/bangumi/person/:id',
-            },
-            {
-                title: '用户日志',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/user/:id',
-                target: '/bangumi/user/blog/:id',
-            },
-            {
-                title: '条目的讨论',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/subject/:id',
-                target: '/bangumi/subject/:id/topics',
-            },
-            {
-                title: '条目的评论',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/subject/:id',
-                target: '/bangumi/subject/:id/blogs',
-            },
-            {
-                title: '条目的章节',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/subject/:id',
-                target: '/bangumi/subject/:id',
-            },
-            {
-                title: '条目的吐槽箱',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/subject/:id',
-                target: '/bangumi/subject/:id/comments',
-            },
-            {
-                title: '放送列表',
-                docs: 'https://docs.rsshub.app/anime.html#bangumi',
-                source: '/calendar',
-                target: '/bangumi/calendar/today',
-            },
-        ],
-    },
     'e-hentai.org/': {
         _name: 'E-Hentai',
         '.': [