rsshub 1.0.0-master.fdf137b → 1.0.0-master.fe775f5

package/lib/config.js

@@ -25,83 +25,38 @@ const calculateValue = () => {
     }
 
     value = {
+        // app config
+        disallowRobot: envs.DISALLOW_ROBOT !== '0' && envs.DISALLOW_ROBOT !== 'false',
+        enableCluster: envs.ENABLE_CLUSTER,
         isPackage: envs.IS_PACKAGE,
-        noLogfiles: envs.NO_LOGFILES,
+        nodeName: envs.NODE_NAME,
+        puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT,
+        // network
         connect: {
             port: envs.PORT || 1200, // 监听端口
             socket: envs.SOCKET || null, // 监听 Unix Socket, null 为禁用
         },
+        listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1
+        requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数
+        requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request
+        ua: envs.UA || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
+        // cors request
+        allowOrigin: envs.ALLOW_ORIGIN,
+        // cache
         cache: {
             type: typeof envs.CACHE_TYPE === 'undefined' ? 'memory' : envs.CACHE_TYPE, // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存
             requestTimeout: parseInt(envs.CACHE_REQUEST_TIMEOUT) || 60,
             routeExpire: parseInt(envs.CACHE_EXPIRE) || 5 * 60, // 路由缓存时间，单位为秒
             contentExpire: parseInt(envs.CACHE_CONTENT_EXPIRE) || 1 * 60 * 60, // 不变内容缓存时间，单位为秒
         },
-        ua: envs.UA || 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.181 Safari/537.36',
-        listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1
-        requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数
-        requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request
-        // 是否显示 Debug 信息，取值 'true' 'false' 'some_string' ，取值为 'true' 时永久显示，取值为 'false' 时永远隐藏，取值为 'some_string' 时请求带上 '?debug=some_string' 显示
-        debugInfo: envs.DEBUG_INFO || 'true',
-        disallowRobot: envs.DISALLOW_ROBOT !== '0' && envs.DISALLOW_ROBOT !== 'false',
-        titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
+        memory: {
+            max: parseInt(envs.MEMORY_MAX) || Math.pow(2, 8), // The maximum number of items that remain in the cache. This must be a positive finite intger.
+            // https://github.com/isaacs/node-lru-cache#options
+        },
         redis: {
             url: envs.REDIS_URL || 'redis://localhost:6379/',
         },
-        nodeName: envs.NODE_NAME,
-        suffix: envs.SUFFIX,
-        pixiv: {
-            refreshToken: envs.PIXIV_REFRESHTOKEN,
-            bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
-            bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
-            bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
-            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
-        },
-        fanbox: {
-            session: envs.FANBOX_SESSION_ID,
-        },
-        disqus: {
-            api_key: envs.DISQUS_API_KEY,
-        },
-        spotify: {
-            clientId: envs.SPOTIFY_CLIENT_ID,
-            clientSecret: envs.SPOTIFY_CLIENT_SECRET,
-            refreshToken: envs.SPOTIFY_REFRESHTOKEN,
-        },
-        twitter: {
-            consumer_key: envs.TWITTER_CONSUMER_KEY,
-            consumer_secret: envs.TWITTER_CONSUMER_SECRET,
-            tokens: twitter_tokens,
-            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
-            // 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
-        },
-        youtube: {
-            key: envs.YOUTUBE_KEY,
-        },
-        telegram: {
-            token: envs.TELEGRAM_TOKEN,
-        },
-        github: {
-            access_token: envs.GITHUB_ACCESS_TOKEN,
-        },
-        authentication: {
-            name: envs.HTTP_BASIC_AUTH_NAME || 'usernam3',
-            pass: envs.HTTP_BASIC_AUTH_PASS || 'passw0rd',
-        },
-        bilibili: {
-            cookies: bilibili_cookies,
-        },
-        yuque: {
-            token: envs.YUQUE_TOKEN,
-        },
-        zhihu: {
-            cookies: envs.ZHIHU_COOKIES,
-        },
-        ncm: {
-            cookies: envs.NCM_COOKIES || '',
-        },
-        puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT,
-        loggerLevel: envs.LOGGER_LEVEL || 'info',
+        // proxy
         proxyUri: envs.PROXY_URI,
         proxy: {
             protocol: envs.PROXY_PROTOCOL,
@@ -110,26 +65,74 @@ const calculateValue = () => {
             auth: envs.PROXY_AUTH,
             url_regex: envs.PROXY_URL_REGEX || '.*',
         },
-        allowOrigin: envs.ALLOW_ORIGIN,
+        // auth
+        authentication: {
+            name: envs.HTTP_BASIC_AUTH_NAME || 'usernam3',
+            pass: envs.HTTP_BASIC_AUTH_PASS || 'passw0rd',
+        },
+        // access control
         blacklist: envs.BLACKLIST && envs.BLACKLIST.split(','),
         whitelist: envs.WHITELIST && envs.WHITELIST.split(','),
         allowLocalhost: envs.ALLOW_LOCALHOST,
         accessKey: envs.ACCESS_KEY,
-        enableCluster: envs.ENABLE_CLUSTER,
-        email: {
-            config: email_config,
-        },
+        // logging
+        // 是否显示 Debug 信息，取值 'true' 'false' 'some_string' ，取值为 'true' 时永久显示，取值为 'false' 时永远隐藏，取值为 'some_string' 时请求带上 '?debug=some_string' 显示
+        debugInfo: envs.DEBUG_INFO || 'true',
+        loggerLevel: envs.LOGGER_LEVEL || 'info',
+        noLogfiles: envs.NO_LOGFILES,
         sentry: {
             dsn: envs.SENTRY,
             routeTimeout: parseInt(envs.SENTRY_ROUTE_TIMEOUT) || 30000,
         },
+        // feed config
+        hotlink: {
+            template: envs.HOTLINK_TEMPLATE,
+        },
+        suffix: envs.SUFFIX,
+        titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
+
+        // Route-specific Configurations
+        bilibili: {
+            cookies: bilibili_cookies,
+        },
+        bitbucket: {
+            username: envs.BITBUCKET_USERNAME,
+            password: envs.BITBUCKET_PASSWORD,
+        },
+        btbyr: {
+            host: envs.BTBYR_HOST,
+            cookies: envs.BTBYR_COOKIE,
+        },
+        bupt: {
+            portal_cookie: envs.BUPT_PORTAL_COOKIE,
+        },
         chuiniu: {
             member: envs.CHUINIU_MEMBER,
         },
-        weibo: {
-            app_key: envs.WEIBO_APP_KEY,
-            app_secret: envs.WEIBO_APP_SECRET,
-            redirect_url: envs.WEIBO_REDIRECT_URL,
+        dida365: {
+            username: envs.DIDA365_USERNAME,
+            password: envs.DIDA365_PASSWORD,
+        },
+        discuz: {
+            cookies: discuz_cookies,
+        },
+        disqus: {
+            api_key: envs.DISQUS_API_KEY,
+        },
+        douban: {
+            cookie: envs.DOUBAN_COOKIE,
+        },
+        ehentai: {
+            ipb_member_id: envs.EH_IPB_MEMBER_ID,
+            ipb_pass_hash: envs.EH_IPB_PASS_HASH,
+            sk: envs.EH_SK,
+            igneous: envs.EH_IGNEOUS,
+        },
+        email: {
+            config: email_config,
+        },
+        fanbox: {
+            session: envs.FANBOX_SESSION_ID,
         },
         fanfou: {
             consumer_key: envs.FANFOU_CONSUMER_KEY,
@@ -137,27 +140,17 @@ const calculateValue = () => {
             username: envs.FANFOU_USERNAME,
             password: envs.FANFOU_PASSWORD,
         },
-        lastfm: {
-            api_key: envs.LASTFM_API_KEY,
-        },
-        pkubbs: {
-            cookie: envs.PKUBBS_COOKIE,
-        },
-        bupt: {
-            portal_cookie: envs.BUPT_PORTAL_COOKIE,
-        },
-        nhentai: {
-            username: envs.NHENTAI_USERNAME,
-            password: envs.NHENTAI_PASSWORD,
+        game4399: {
+            cookie: envs.GAME_4399,
         },
-        discuz: {
-            cookies: discuz_cookies,
+        github: {
+            access_token: envs.GITHUB_ACCESS_TOKEN,
         },
-        scihub: {
-            host: envs.SCIHUB_HOST || 'https://sci-hub.se/',
+        google: {
+            fontsApiKey: envs.GOOGLE_FONTS_API_KEY,
         },
-        hotlink: {
-            template: envs.HOTLINK_TEMPLATE,
+        infzm: {
+            cookie: envs.INFZM_COOKIE,
         },
         initium: {
             username: envs.INITIUM_USERNAME,
@@ -165,59 +158,94 @@ const calculateValue = () => {
             bearertoken: envs.INITIUM_BEARER_TOKEN,
             iap_receipt: envs.INITIUM_IAP_RECEIPT,
         },
-        btbyr: {
-            host: envs.BTBYR_HOST,
-            cookies: envs.BTBYR_COOKIE,
+        instagram: {
+            username: envs.IG_USERNAME,
+            password: envs.IG_PASSWORD,
+            proxy: envs.IG_PROXY,
+        },
+        lastfm: {
+            api_key: envs.LASTFM_API_KEY,
         },
         mastodon: {
             apiHost: envs.MASTODON_API_HOST,
             accessToken: envs.MASTODON_API_ACCESS_TOKEN,
             acctDomain: envs.MASTODON_API_ACCT_DOMAIN,
         },
-        xiaoyuzhou: {
-            device_id: envs.XIAOYUZHOU_ID,
-            refresh_token: envs.XIAOYUZHOU_TOKEN,
-        },
         miniflux: {
             instance: envs.MINIFLUX_INSTANCE || 'https://reader.miniflux.app',
             token: envs.MINIFLUX_TOKEN || '',
         },
-        nga: {
-            uid: envs.NGA_PASSPORT_UID,
-            cid: envs.NGA_PASSPORT_CID,
+        ncm: {
+            cookies: envs.NCM_COOKIES || '',
         },
         newrank: {
             cookie: envs.NEWRANK_COOKIE,
         },
-        ximalaya: {
-            token: envs.XIMALAYA_TOKEN,
+        nga: {
+            uid: envs.NGA_PASSPORT_UID,
+            cid: envs.NGA_PASSPORT_CID,
         },
-        game4399: {
-            cookie: envs.GAME_4399,
+        nhentai: {
+            username: envs.NHENTAI_USERNAME,
+            password: envs.NHENTAI_PASSWORD,
         },
-        dida365: {
-            username: envs.DIDA365_USERNAME,
-            password: envs.DIDA365_PASSWORD,
+        pixiv: {
+            refreshToken: envs.PIXIV_REFRESHTOKEN,
+            bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
+            bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
+            bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
+            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
+        },
+        pkubbs: {
+            cookie: envs.PKUBBS_COOKIE,
         },
         scboy: {
             token: envs.SCBOY_BBS_TOKEN,
         },
-        wordpress: {
-            cdnUrl: envs.WORDPRESS_CDN,
+        scihub: {
+            host: envs.SCIHUB_HOST || 'https://sci-hub.se/',
         },
-        ehentai: {
-            ipb_member_id: envs.EH_IPB_MEMBER_ID,
-            ipb_pass_hash: envs.EH_IPB_PASS_HASH,
-            sk: envs.EH_SK,
+        spotify: {
+            clientId: envs.SPOTIFY_CLIENT_ID,
+            clientSecret: envs.SPOTIFY_CLIENT_SECRET,
+            refreshToken: envs.SPOTIFY_REFRESHTOKEN,
         },
-        infzm: {
-            cookie: envs.INFZM_COOKIE,
+        telegram: {
+            token: envs.TELEGRAM_TOKEN,
+        },
+        twitter: {
+            consumer_key: envs.TWITTER_CONSUMER_KEY,
+            consumer_secret: envs.TWITTER_CONSUMER_SECRET,
+            tokens: twitter_tokens,
+            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+            // 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
+        },
+        weibo: {
+            app_key: envs.WEIBO_APP_KEY,
+            app_secret: envs.WEIBO_APP_SECRET,
+            redirect_url: envs.WEIBO_REDIRECT_URL,
         },
         wenku8: {
             cookie: envs.WENKU8_COOKIE,
         },
-        google: {
-            fontsApiKey: envs.GOOGLE_FONTS_API_KEY,
+        wordpress: {
+            cdnUrl: envs.WORDPRESS_CDN,
+        },
+        xiaoyuzhou: {
+            device_id: envs.XIAOYUZHOU_ID,
+            refresh_token: envs.XIAOYUZHOU_TOKEN,
+        },
+        ximalaya: {
+            token: envs.XIMALAYA_TOKEN,
+        },
+        youtube: {
+            key: envs.YOUTUBE_KEY,
+        },
+        yuque: {
+            token: envs.YUQUE_TOKEN,
+        },
+        zhihu: {
+            cookies: envs.ZHIHU_COOKIES,
         },
     };
 };

package/lib/middleware/cache/index.js

@@ -41,7 +41,7 @@ if (config.cache.type === 'redis') {
             value = JSON.stringify(value);
         }
         if (key) {
-            return pageCache.set(key, value, maxAge * 1000);
+            return pageCache.set(key, value, { ttl: maxAge * 1000 });
         }
     };
 } else {
@@ -55,8 +55,8 @@ module.exports = function (app) {
     const { get, set, status } = cacheModule;
     app.context.cache = {
         ...cacheModule,
-        tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire) => {
-            let v = await get(key);
+        tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire, refresh = true) => {
+            let v = await get(key, refresh);
             if (!v) {
                 v = await getValueFunc();
                 set(key, v, maxAge);

package/lib/middleware/cache/memory.js

@@ -4,13 +4,13 @@ const config = require('@/config').value;
 const status = { available: false };
 
 const pageCache = new Lru({
-    maxAge: config.cache.routeExpire * 1000,
-    max: Infinity,
+    ttl: config.cache.routeExpire * 1000,
+    max: config.memory.max,
 });
 
 const routeCache = new Lru({
-    maxAge: config.cache.routeExpire * 1000,
-    max: Infinity,
+    ttl: config.cache.routeExpire * 1000,
+    max: config.memory.max,
     updateAgeOnGet: true,
 });
 
@@ -34,7 +34,7 @@ module.exports = {
             value = JSON.stringify(value);
         }
         if (key && status.available) {
-            return (refresh ? routeCache : pageCache).set(key, value, maxAge * 1000);
+            return (refresh ? routeCache : pageCache).set(key, value, { ttl: maxAge * 1000 });
         }
     },
     clients: { pageCache, routeCache },

package/lib/middleware/cache/redis.js

@@ -18,12 +18,26 @@ redisClient.on('connect', () => {
     logger.info('Redis connected.');
 });
 
+const getCacheTtlKey = (key) => {
+    if (key.startsWith('cacheTtl:')) {
+        throw Error('"cacheTtl:" prefix is reserved for the internal usage, please change your cache key'); // blocking any attempt to get/set the cacheTtl
+    }
+    return `cacheTtl:${key}`;
+};
+
 module.exports = {
     get: async (key, refresh = true) => {
         if (key && status.available) {
-            let value = await redisClient.get(key);
+            const cacheTtlKey = getCacheTtlKey(key);
+            let [value, cacheTtl] = await redisClient.mget(key, cacheTtlKey);
             if (value && refresh) {
-                redisClient.expire(key, config.cache.contentExpire);
+                if (!cacheTtl) {
+                    cacheTtl = config.cache.contentExpire;
+                    redisClient.set(cacheTtlKey, cacheTtl, 'EX', cacheTtl);
+                } else {
+                    redisClient.expire(cacheTtlKey, cacheTtl);
+                }
+                redisClient.expire(key, cacheTtl);
                 value = value + '';
             }
             return value;
@@ -40,6 +54,7 @@ module.exports = {
             value = JSON.stringify(value);
         }
         if (key) {
+            redisClient.set(getCacheTtlKey(key), maxAge, 'EX', maxAge);
             return redisClient.set(key, value, 'EX', maxAge); // setMode: https://redis.io/commands/set
         }
     },

package/lib/middleware/template.js

@@ -54,10 +54,10 @@ module.exports = async (ctx, next) => {
                         }
                     }
 
-                    if (item.enclosure_length) {
-                        const itunes_duration =
-                            Math.floor(item.enclosure_length / 3600) + ':' + (Math.floor((item.enclosure_length % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.enclosure_length % 3600) % 60) / 100).toFixed(2).slice(-2);
-                        item.itunes_duration = itunes_duration;
+                    if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && item.itunes_duration.indexOf(':') === -1) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) {
+                        item.itunes_duration = +item.itunes_duration;
+                        item.itunes_duration =
+                            Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2);
                     }
                 });
         }

package/lib/protected_router.js

@@ -6,6 +6,6 @@ const config = require('./config').value;
 router.use('/(.*)', auth(config.authentication));
 
 // RSSHub
-router.get('/rsshub/routes', require('./routes/rsshub/routes'));
+router.get('/rsshub/routes/:lang?', require('./v2/rsshub/routes'));
 
 module.exports = router;