rsshub 1.0.0-master.fe69819 → 1.0.0-master.febd623

package/README.md

@@ -5,9 +5,11 @@
 
 > 🍰 Everything is RSSible
 
-[![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?style=flat-square)](https://t.me/rsshub)
-[![build status](https://img.shields.io/travis/DIYgod/RSSHub/master.svg?style=flat-square)](https://travis-ci.org/DIYgod/RSSHub)
-[![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square)](https://codecov.io/github/DIYgod/RSSHub?branch=master)
+[![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?logo=telegram&style=flat-square)](https://t.me/rsshub)
+[![npm publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/publish/master?label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
+[![docker publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/%5Bdocker%5D%20CI%20for%20releases/master?label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
+[![test](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/test/master?label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
+[![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square&logo=codecov)](https://app.codecov.io/gh/DIYgod/RSSHub/branch/master)
 
 ## Introduction
 
@@ -37,7 +39,7 @@ RSSHub 是一个开源、简单易用、易于扩展的 RSS 生成器，可以
 
 ### Sponsors
 
-[Sayori Studio](https://t.me/SayoriStudio) . [Sion Kazama](https://blog.sion.moe) . [琚致远](https://wineso.me/) . [Rolly RSS 阅读器](https://www.coolapk.com/apk/239500) . [mokeyjay](https://www.mokeyjay.com/) . [萌开源联盟](https://www.moeunion.com) . [hooke007](https://github.com/hooke007/MPV_lazy) . [feeds.pub](https://feeds.pub)
+[Sayori Studio](https://t.me/SayoriStudio) . [Sion Kazama](https://blog.sion.moe) . [琚致远](https://wineso.me/) . [Rolly RSS 阅读器](https://www.coolapk.com/apk/239500) . [mokeyjay](https://www.mokeyjay.com/) . [萌开源联盟](https://www.moeunion.com) . [hooke007](https://github.com/hooke007/MPV_lazy) . [feeds.pub](https://feeds.pub) . [KINGX@安全引擎](http://cve.today/)
 
 [![](https://opencollective.com/static/images/become_sponsor.svg)](https://docs.rsshub.app/en/support/)
 

package/lib/config.js

@@ -1,7 +1,7 @@
 require('dotenv').config();
+const randUserAgent = require('./utils/rand-user-agent');
 let envs = process.env;
 let value;
-
 const calculateValue = () => {
     const bilibili_cookies = {};
     const twitter_tokens = {};
@@ -25,83 +25,39 @@ const calculateValue = () => {
     }
 
     value = {
+        // app config
+        disallowRobot: envs.DISALLOW_ROBOT !== '0' && envs.DISALLOW_ROBOT !== 'false',
+        enableCluster: envs.ENABLE_CLUSTER,
         isPackage: envs.IS_PACKAGE,
-        noLogfiles: envs.NO_LOGFILES,
+        nodeName: envs.NODE_NAME,
+        puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT,
+        chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH,
+        // network
         connect: {
             port: envs.PORT || 1200, // 监听端口
             socket: envs.SOCKET || null, // 监听 Unix Socket, null 为禁用
         },
+        listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1
+        requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数
+        requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request
+        ua: envs.UA || randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }),
+        // cors request
+        allowOrigin: envs.ALLOW_ORIGIN,
+        // cache
         cache: {
             type: typeof envs.CACHE_TYPE === 'undefined' ? 'memory' : envs.CACHE_TYPE, // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存
             requestTimeout: parseInt(envs.CACHE_REQUEST_TIMEOUT) || 60,
             routeExpire: parseInt(envs.CACHE_EXPIRE) || 5 * 60, // 路由缓存时间，单位为秒
             contentExpire: parseInt(envs.CACHE_CONTENT_EXPIRE) || 1 * 60 * 60, // 不变内容缓存时间，单位为秒
         },
-        ua: envs.UA || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
-        listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1
-        requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数
-        requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request
-        // 是否显示 Debug 信息，取值 'true' 'false' 'some_string' ，取值为 'true' 时永久显示，取值为 'false' 时永远隐藏，取值为 'some_string' 时请求带上 '?debug=some_string' 显示
-        debugInfo: envs.DEBUG_INFO || 'true',
-        disallowRobot: envs.DISALLOW_ROBOT !== '0' && envs.DISALLOW_ROBOT !== 'false',
-        titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
+        memory: {
+            max: parseInt(envs.MEMORY_MAX) || Math.pow(2, 8), // The maximum number of items that remain in the cache. This must be a positive finite intger.
+            // https://github.com/isaacs/node-lru-cache#options
+        },
         redis: {
             url: envs.REDIS_URL || 'redis://localhost:6379/',
         },
-        nodeName: envs.NODE_NAME,
-        suffix: envs.SUFFIX,
-        pixiv: {
-            refreshToken: envs.PIXIV_REFRESHTOKEN,
-            bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
-            bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
-            bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
-            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
-        },
-        fanbox: {
-            session: envs.FANBOX_SESSION_ID,
-        },
-        disqus: {
-            api_key: envs.DISQUS_API_KEY,
-        },
-        spotify: {
-            clientId: envs.SPOTIFY_CLIENT_ID,
-            clientSecret: envs.SPOTIFY_CLIENT_SECRET,
-            refreshToken: envs.SPOTIFY_REFRESHTOKEN,
-        },
-        twitter: {
-            consumer_key: envs.TWITTER_CONSUMER_KEY,
-            consumer_secret: envs.TWITTER_CONSUMER_SECRET,
-            tokens: twitter_tokens,
-            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
-            // 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
-        },
-        youtube: {
-            key: envs.YOUTUBE_KEY,
-        },
-        telegram: {
-            token: envs.TELEGRAM_TOKEN,
-        },
-        github: {
-            access_token: envs.GITHUB_ACCESS_TOKEN,
-        },
-        authentication: {
-            name: envs.HTTP_BASIC_AUTH_NAME || 'usernam3',
-            pass: envs.HTTP_BASIC_AUTH_PASS || 'passw0rd',
-        },
-        bilibili: {
-            cookies: bilibili_cookies,
-        },
-        yuque: {
-            token: envs.YUQUE_TOKEN,
-        },
-        zhihu: {
-            cookies: envs.ZHIHU_COOKIES,
-        },
-        ncm: {
-            cookies: envs.NCM_COOKIES || '',
-        },
-        puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT,
-        loggerLevel: envs.LOGGER_LEVEL || 'info',
+        // proxy
         proxyUri: envs.PROXY_URI,
         proxy: {
             protocol: envs.PROXY_PROTOCOL,
@@ -110,26 +66,77 @@ const calculateValue = () => {
             auth: envs.PROXY_AUTH,
             url_regex: envs.PROXY_URL_REGEX || '.*',
         },
-        allowOrigin: envs.ALLOW_ORIGIN,
+        // auth
+        authentication: {
+            name: envs.HTTP_BASIC_AUTH_NAME || 'usernam3',
+            pass: envs.HTTP_BASIC_AUTH_PASS || 'passw0rd',
+        },
+        // access control
         blacklist: envs.BLACKLIST && envs.BLACKLIST.split(','),
         whitelist: envs.WHITELIST && envs.WHITELIST.split(','),
         allowLocalhost: envs.ALLOW_LOCALHOST,
         accessKey: envs.ACCESS_KEY,
-        enableCluster: envs.ENABLE_CLUSTER,
-        email: {
-            config: email_config,
-        },
+        // logging
+        // 是否显示 Debug 信息，取值 'true' 'false' 'some_string' ，取值为 'true' 时永久显示，取值为 'false' 时永远隐藏，取值为 'some_string' 时请求带上 '?debug=some_string' 显示
+        debugInfo: envs.DEBUG_INFO || 'true',
+        loggerLevel: envs.LOGGER_LEVEL || 'info',
+        noLogfiles: envs.NO_LOGFILES,
         sentry: {
             dsn: envs.SENTRY,
             routeTimeout: parseInt(envs.SENTRY_ROUTE_TIMEOUT) || 30000,
         },
+        // feed config
+        hotlink: {
+            template: envs.HOTLINK_TEMPLATE,
+            includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','),
+            excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','),
+        },
+        suffix: envs.SUFFIX,
+        titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
+
+        // Route-specific Configurations
+        bilibili: {
+            cookies: bilibili_cookies,
+        },
+        bitbucket: {
+            username: envs.BITBUCKET_USERNAME,
+            password: envs.BITBUCKET_PASSWORD,
+        },
+        btbyr: {
+            host: envs.BTBYR_HOST,
+            cookies: envs.BTBYR_COOKIE,
+        },
+        bupt: {
+            portal_cookie: envs.BUPT_PORTAL_COOKIE,
+        },
         chuiniu: {
             member: envs.CHUINIU_MEMBER,
         },
-        weibo: {
-            app_key: envs.WEIBO_APP_KEY,
-            app_secret: envs.WEIBO_APP_SECRET,
-            redirect_url: envs.WEIBO_REDIRECT_URL,
+        dida365: {
+            username: envs.DIDA365_USERNAME,
+            password: envs.DIDA365_PASSWORD,
+        },
+        discuz: {
+            cookies: discuz_cookies,
+        },
+        disqus: {
+            api_key: envs.DISQUS_API_KEY,
+        },
+        douban: {
+            cookie: envs.DOUBAN_COOKIE,
+        },
+        ehentai: {
+            ipb_member_id: envs.EH_IPB_MEMBER_ID,
+            ipb_pass_hash: envs.EH_IPB_PASS_HASH,
+            sk: envs.EH_SK,
+            igneous: envs.EH_IGNEOUS,
+            img_proxy: envs.EH_IMG_PROXY,
+        },
+        email: {
+            config: email_config,
+        },
+        fanbox: {
+            session: envs.FANBOX_SESSION_ID,
         },
         fanfou: {
             consumer_key: envs.FANFOU_CONSUMER_KEY,
@@ -137,27 +144,24 @@ const calculateValue = () => {
             username: envs.FANFOU_USERNAME,
             password: envs.FANFOU_PASSWORD,
         },
-        lastfm: {
-            api_key: envs.LASTFM_API_KEY,
-        },
-        pkubbs: {
-            cookie: envs.PKUBBS_COOKIE,
+        game4399: {
+            cookie: envs.GAME_4399,
         },
-        bupt: {
-            portal_cookie: envs.BUPT_PORTAL_COOKIE,
+        github: {
+            access_token: envs.GITHUB_ACCESS_TOKEN,
         },
-        nhentai: {
-            username: envs.NHENTAI_USERNAME,
-            password: envs.NHENTAI_PASSWORD,
+        gitee: {
+            access_token: envs.GITEE_ACCESS_TOKEN,
         },
-        discuz: {
-            cookies: discuz_cookies,
+        google: {
+            fontsApiKey: envs.GOOGLE_FONTS_API_KEY,
         },
-        scihub: {
-            host: envs.SCIHUB_HOST || 'https://sci-hub.se/',
+        hefeng: {
+            // weather
+            key: envs.HEFENG_KEY,
         },
-        hotlink: {
-            template: envs.HOTLINK_TEMPLATE,
+        infzm: {
+            cookie: envs.INFZM_COOKIE,
         },
         initium: {
             username: envs.INITIUM_USERNAME,
@@ -165,64 +169,100 @@ const calculateValue = () => {
             bearertoken: envs.INITIUM_BEARER_TOKEN,
             iap_receipt: envs.INITIUM_IAP_RECEIPT,
         },
-        btbyr: {
-            host: envs.BTBYR_HOST,
-            cookies: envs.BTBYR_COOKIE,
+        instagram: {
+            username: envs.IG_USERNAME,
+            password: envs.IG_PASSWORD,
+            proxy: envs.IG_PROXY,
+        },
+        lastfm: {
+            api_key: envs.LASTFM_API_KEY,
         },
         mastodon: {
             apiHost: envs.MASTODON_API_HOST,
             accessToken: envs.MASTODON_API_ACCESS_TOKEN,
             acctDomain: envs.MASTODON_API_ACCT_DOMAIN,
         },
-        xiaoyuzhou: {
-            device_id: envs.XIAOYUZHOU_ID,
-            refresh_token: envs.XIAOYUZHOU_TOKEN,
-        },
         miniflux: {
             instance: envs.MINIFLUX_INSTANCE || 'https://reader.miniflux.app',
             token: envs.MINIFLUX_TOKEN || '',
         },
+        ncm: {
+            cookies: envs.NCM_COOKIES || '',
+        },
+        newrank: {
+            cookie: envs.NEWRANK_COOKIE,
+        },
         nga: {
             uid: envs.NGA_PASSPORT_UID,
             cid: envs.NGA_PASSPORT_CID,
         },
-        newrank: {
-            cookie: envs.NEWRANK_COOKIE,
+        nhentai: {
+            username: envs.NHENTAI_USERNAME,
+            password: envs.NHENTAI_PASSWORD,
         },
-        ximalaya: {
-            token: envs.XIMALAYA_TOKEN,
+        pianyuan: {
+            cookie: envs.PIANYUAN_COOKIE,
         },
-        game4399: {
-            cookie: envs.GAME_4399,
+        pixiv: {
+            refreshToken: envs.PIXIV_REFRESHTOKEN,
+            bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
+            bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
+            bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
+            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
         },
-        dida365: {
-            username: envs.DIDA365_USERNAME,
-            password: envs.DIDA365_PASSWORD,
+        pkubbs: {
+            cookie: envs.PKUBBS_COOKIE,
         },
         scboy: {
             token: envs.SCBOY_BBS_TOKEN,
         },
-        wordpress: {
-            cdnUrl: envs.WORDPRESS_CDN,
+        scihub: {
+            host: envs.SCIHUB_HOST || 'https://sci-hub.se/',
         },
-        ehentai: {
-            ipb_member_id: envs.EH_IPB_MEMBER_ID,
-            ipb_pass_hash: envs.EH_IPB_PASS_HASH,
-            sk: envs.EH_SK,
+        spotify: {
+            clientId: envs.SPOTIFY_CLIENT_ID,
+            clientSecret: envs.SPOTIFY_CLIENT_SECRET,
+            refreshToken: envs.SPOTIFY_REFRESHTOKEN,
         },
-        infzm: {
-            cookie: envs.INFZM_COOKIE,
+        telegram: {
+            token: envs.TELEGRAM_TOKEN,
+        },
+        twitter: {
+            consumer_key: envs.TWITTER_CONSUMER_KEY,
+            consumer_secret: envs.TWITTER_CONSUMER_SECRET,
+            tokens: twitter_tokens,
+            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+            // 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
+        },
+        weibo: {
+            app_key: envs.WEIBO_APP_KEY,
+            app_secret: envs.WEIBO_APP_SECRET,
+            redirect_url: envs.WEIBO_REDIRECT_URL,
         },
         wenku8: {
             cookie: envs.WENKU8_COOKIE,
         },
-        google: {
-            fontsApiKey: envs.GOOGLE_FONTS_API_KEY,
+        wordpress: {
+            cdnUrl: envs.WORDPRESS_CDN,
         },
-        instagram: {
-            username: envs.IG_USERNAME,
-            password: envs.IG_PASSWORD,
-            proxy: envs.IG_PROXY,
+        xiaoyuzhou: {
+            device_id: envs.XIAOYUZHOU_ID,
+            refresh_token: envs.XIAOYUZHOU_TOKEN,
+        },
+        ximalaya: {
+            token: envs.XIMALAYA_TOKEN,
+        },
+        youtube: {
+            key: envs.YOUTUBE_KEY,
+            clientId: envs.YOUTUBE_CLIENT_ID,
+            clientSecret: envs.YOUTUBE_CLIENT_SECRET,
+            refreshToken: envs.YOUTUBE_REFRESH_TOKEN,
+        },
+        yuque: {
+            token: envs.YUQUE_TOKEN,
+        },
+        zhihu: {
+            cookies: envs.ZHIHU_COOKIES,
         },
     };
 };

package/lib/middleware/anti-hotlink.js

@@ -2,6 +2,23 @@ const config = require('@/config').value;
 const cheerio = require('cheerio');
 const logger = require('@/utils/logger');
 
+// match path or sub-path
+const matchPath = (path, paths) => {
+    for (const p of paths) {
+        if (path.startsWith(p) && (path.length === p.length || path[p.length] === '/')) {
+            return true;
+        }
+    }
+    return false;
+};
+
+// return ture if the path needs to be processed
+const filterPath = (path) => {
+    const include = config.hotlink.includePaths;
+    const exclude = config.hotlink.excludePaths;
+    return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude));
+};
+
 const interpolate = (str, obj) => str.replace(/\${([^}]+)}/g, (_, prop) => obj[prop]);
 // I don't want to keep another regex and
 // URL will be the standard way to parse URL
@@ -16,7 +33,13 @@ const parseUrl = (str) => {
     return url;
 };
 const replaceUrls = (body, template) => {
-    const $ = cheerio.load(body, { decodeEntities: false, xmlMode: true });
+    // const $ = cheerio.load(body, { decodeEntities: false, xmlMode: true });
+    // `<br><img><hr><video><source>abc</video>` => `<br><img><hr><video><source>abc</source></video></hr></img></br>`
+    // so awful...
+    // "In HTML, using a closing tag on an empty element is usually invalid."
+    // https://developer.mozilla.org/en-US/docs/Glossary/Empty_element
+    // I guess it is just a workaround to drop `<html><head></head><body>`, so this is what we exactly need:
+    const $ = cheerio.load(body, null, false);
     $('img').each(function () {
         const old_src = $(this).attr('src');
         const url = parseUrl(old_src);
@@ -33,22 +56,25 @@ module.exports = async (ctx, next) => {
     await next();
 
     const template = config.hotlink.template;
+
+    if (!template || !filterPath(ctx.request.path)) {
+        return;
+    }
+
     // Assume that only description include image link
     // and here we will only check them in description.
-    // Use Cherrio to load the description as html and filter all
+    // Use Cheerio to load the description as html and filter all
     // image link
-    if (template) {
-        if (ctx.state.data) {
-            if (ctx.state.data.description) {
-                ctx.state.data.description = replaceUrls(ctx.state.data.description, template);
-            }
-
-            ctx.state.data.item &&
-                ctx.state.data.item.forEach((item) => {
-                    if (item.description) {
-                        item.description = replaceUrls(item.description, template);
-                    }
-                });
+    if (ctx.state.data) {
+        if (ctx.state.data.description) {
+            ctx.state.data.description = replaceUrls(ctx.state.data.description, template);
         }
+
+        ctx.state.data.item &&
+            ctx.state.data.item.forEach((item) => {
+                if (item.description) {
+                    item.description = replaceUrls(item.description, template);
+                }
+            });
     }
 };

package/lib/middleware/cache/index.js

@@ -27,10 +27,10 @@ if (config.cache.type === 'redis') {
     globalCache.set = cacheModule.set;
 } else if (config.cache.type === 'memory') {
     cacheModule = require('./memory');
-    const { pageCache } = cacheModule.clients;
+    const { memoryCache } = cacheModule.clients;
     globalCache.get = (key) => {
         if (key && cacheModule.status.available) {
-            return pageCache.get(key);
+            return memoryCache.get(key, { updateAgeOnGet: false });
         }
     };
     globalCache.set = (key, value, maxAge) => {
@@ -41,7 +41,7 @@ if (config.cache.type === 'redis') {
             value = JSON.stringify(value);
         }
         if (key) {
-            return pageCache.set(key, value, maxAge * 1000);
+            return memoryCache.set(key, value, { ttl: maxAge * 1000 });
         }
     };
 } else {
@@ -55,8 +55,8 @@ module.exports = function (app) {
     const { get, set, status } = cacheModule;
     app.context.cache = {
         ...cacheModule,
-        tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire) => {
-            let v = await get(key);
+        tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire, refresh = true) => {
+            let v = await get(key, refresh);
             if (!v) {
                 v = await getValueFunc();
                 set(key, v, maxAge);

package/lib/middleware/cache/memory.js

@@ -3,15 +3,9 @@ const config = require('@/config').value;
 
 const status = { available: false };
 
-const pageCache = new Lru({
-    maxAge: config.cache.routeExpire * 1000,
-    max: Infinity,
-});
-
-const routeCache = new Lru({
-    maxAge: config.cache.routeExpire * 1000,
-    max: Infinity,
-    updateAgeOnGet: true,
+const memoryCache = new Lru({
+    ttl: config.cache.routeExpire * 1000,
+    max: config.memory.max,
 });
 
 status.available = true;
@@ -19,14 +13,14 @@ status.available = true;
 module.exports = {
     get: (key, refresh = true) => {
         if (key && status.available) {
-            let value = (refresh ? routeCache : pageCache).get(key);
+            let value = memoryCache.get(key, { updateAgeOnGet: refresh });
             if (value) {
                 value = value + '';
             }
             return value;
         }
     },
-    set: (key, value, maxAge = config.cache.contentExpire, refresh = true) => {
+    set: (key, value, maxAge = config.cache.contentExpire) => {
         if (!value || value === 'undefined') {
             value = '';
         }
@@ -34,9 +28,9 @@ module.exports = {
             value = JSON.stringify(value);
         }
         if (key && status.available) {
-            return (refresh ? routeCache : pageCache).set(key, value, maxAge * 1000);
+            return memoryCache.set(key, value, { ttl: maxAge * 1000 });
         }
     },
-    clients: { pageCache, routeCache },
+    clients: { memoryCache },
     status,
 };

package/lib/middleware/cache/redis.js

@@ -18,12 +18,28 @@ redisClient.on('connect', () => {
     logger.info('Redis connected.');
 });
 
+const getCacheTtlKey = (key) => {
+    if (key.startsWith('cacheTtl:')) {
+        throw Error('"cacheTtl:" prefix is reserved for the internal usage, please change your cache key'); // blocking any attempt to get/set the cacheTtl
+    }
+    return `cacheTtl:${key}`;
+};
+
 module.exports = {
     get: async (key, refresh = true) => {
         if (key && status.available) {
-            let value = await redisClient.get(key);
+            const cacheTtlKey = getCacheTtlKey(key);
+            let [value, cacheTtl] = await redisClient.mget(key, cacheTtlKey);
             if (value && refresh) {
-                redisClient.expire(key, config.cache.contentExpire);
+                if (!cacheTtl) {
+                    // if cacheTtl is not set, that means the cache expire time is contentExpire
+                    cacheTtl = config.cache.contentExpire;
+                    // dont save cacheTtl to Redis, as it is the default value
+                    // redisClient.set(cacheTtlKey, cacheTtl, 'EX', cacheTtl);
+                } else {
+                    redisClient.expire(cacheTtlKey, cacheTtl);
+                }
+                redisClient.expire(key, cacheTtl);
                 value = value + '';
             }
             return value;
@@ -40,6 +56,10 @@ module.exports = {
             value = JSON.stringify(value);
         }
         if (key) {
+            if (maxAge !== config.cache.contentExpire) {
+                // Only set cacheTtlKey if maxAge !== contentExpire
+                redisClient.set(getCacheTtlKey(key), maxAge, 'EX', maxAge);
+            }
             return redisClient.set(key, value, 'EX', maxAge); // setMode: https://redis.io/commands/set
         }
     },

package/lib/middleware/onerror.js

@@ -6,6 +6,7 @@ const path = require('path');
 const { RequestInProgressError } = require('@/errors');
 
 let Sentry;
+let gitHash;
 
 if (config.sentry.dsn) {
     Sentry = Sentry || require('@sentry/node');
@@ -19,6 +20,12 @@ if (config.sentry.dsn) {
     logger.info('Sentry inited.');
 }
 
+try {
+    gitHash = require('git-rev-sync').short();
+} catch (e) {
+    gitHash = (process.env.HEROKU_SLUG_COMMIT && process.env.HEROKU_SLUG_COMMIT.slice(0, 7)) || (process.env.VERCEL_GIT_COMMIT_SHA && process.env.VERCEL_GIT_COMMIT_SHA.slice(0, 7)) || 'unknown';
+}
+
 module.exports = async (ctx, next) => {
     try {
         const time = +new Date();
@@ -55,6 +62,7 @@ module.exports = async (ctx, next) => {
             if (err instanceof RequestInProgressError) {
                 ctx.status = 503;
                 message = err.message;
+                ctx.set('Cache-Control', `public, max-age=${config.cache.requestTimeout}`);
             } else if (ctx.status === 403) {
                 message = err.message;
             } else {
@@ -68,6 +76,7 @@ module.exports = async (ctx, next) => {
                 message,
                 errorPath: ctx.path,
                 nodeVersion: process.version,
+                gitHash,
             });
         }
 

package/lib/middleware/parameter.js

@@ -10,9 +10,17 @@ module.exports = async (ctx, next) => {
     await next();
 
     if (!ctx.state.data && !ctx._matchedRoute) {
-        ctx.set({
-            'Cache-Control': `public, max-age=${config.cache.routeExpire * 100}`,
-        });
+        // Given that the official demo has a cache TTL of 2h, a "wrong path" page will be cached by CloudFlare for
+        // 200h (8.33d).
+        // What makes it worse is that the documentation contains status badges to detect the availability of routes,
+        // but the documentation is updated more timely than the official demo, so the every example path of every
+        // new route will probably have a "wrong path" page cached for at least 200h soon after accepted. That is to
+        // say, the example paths of a new route will probably be unavailable on the public demo in the first 200h
+        // after accepted.
+        // As a conclusion, the next 3 lines has been commented out. (exactly the same behavior as any internal error)
+        // ctx.set({
+        //     'Cache-Control': `public, max-age=${config.cache.routeExpire * 100}`,
+        // });
         throw Error('wrong path');
     }