rsshub 1.0.0-master.efcdccc → 1.0.0-master.efe64eb

package/README.md

@@ -32,7 +32,7 @@ RSSHub 是一个开源、简单易用、易于扩展的 RSS 生成器，可以
 ### Special Sponsors
 
 <div>
-<a href="https://rss3.io/" target="_blank"><img height="60px" src="https://i.imgur.com/v7TF0jp.jpg"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="https://sa.net/" target="_blank"><img height="60px" src="https://i.imgur.com/3rM8nWI.png"></a>
+<a href="https://rss3.io/" target="_blank"><img height="60px" src="https://i.imgur.com/v7TF0jp.jpg"></a>
 </div>
 
 ### Sponsors
@@ -85,6 +85,7 @@ You can support RSSHub via donations.
 
 Recurring donors will be rewarded via express issue response, or even have your name displayed on our GitHub page and website.
 
+-   Become a Sponser on [GitHub](https://github.com/sponsors/DIYgod)
 -   Become a Sponser on [Patreon](https://www.patreon.com/DIYgod)
 -   Become a Sponser on [爱发电](https://afdian.net/@diygod)
 -   Contact us directly: i@diygod.me

package/lib/api_router.js

@@ -1,15 +1,13 @@
 const Router = require('@koa/router');
 const router = new Router();
-const routes = require('./router');
+const maintainer = require('./maintainer');
 
 router.get('/routes/:name?', (ctx) => {
-    const allRoutes = Array.from(routes.stack);
-    allRoutes.shift();
     const result = {};
     let counter = 0;
 
-    allRoutes.forEach((i) => {
-        const path = i.path;
+    Object.keys(maintainer).forEach((i) => {
+        const path = i;
         const top = path.split('/')[1];
 
         if (!ctx.params.name || top === ctx.params.name) {

package/lib/app.js

@@ -16,8 +16,10 @@ const favicon = require('koa-favicon');
 const debug = require('./middleware/debug');
 const accessControl = require('./middleware/access-control');
 const antiHotlink = require('./middleware/anti-hotlink');
+const loadOnDemand = require('./middleware/load-on-demand');
 
 const router = require('./router');
+const core_router = require('./core_router');
 const protected_router = require('./protected_router');
 const mount = require('koa-mount');
 
@@ -70,16 +72,21 @@ app.use(antiHotlink);
 // 3 filter content
 app.use(parameter);
 
+// No Cache routes
+app.use(mount('/', core_router.routes())).use(core_router.allowedMethods());
+// API router
+app.use(mount('/api', api_router.routes())).use(api_router.allowedMethods());
+
 // 2 cache
 app.use(cache(app));
 
+// 1 load on demand
+app.use(loadOnDemand(app));
+
 // router
 app.use(mount('/', router.routes())).use(router.allowedMethods());
 
 // routes the require authentication
 app.use(mount('/protected', protected_router.routes())).use(protected_router.allowedMethods());
 
-// API router
-app.use(mount('/api', api_router.routes())).use(api_router.allowedMethods());
-
 module.exports = app;

package/lib/config.js

@@ -33,6 +33,7 @@ const calculateValue = () => {
         },
         cache: {
             type: typeof envs.CACHE_TYPE === 'undefined' ? 'memory' : envs.CACHE_TYPE, // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存
+            requestTimeout: parseInt(envs.CACHE_REQUEST_TIMEOUT) || 60,
             routeExpire: parseInt(envs.CACHE_EXPIRE) || 5 * 60, // 路由缓存时间，单位为秒
             contentExpire: parseInt(envs.CACHE_CONTENT_EXPIRE) || 1 * 60 * 60, // 不变内容缓存时间，单位为秒
         },
@@ -54,6 +55,7 @@ const calculateValue = () => {
             bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
             bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
             bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
+            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
         },
         fanbox: {
             session: envs.FANBOX_SESSION_ID,
@@ -65,6 +67,8 @@ const calculateValue = () => {
             consumer_key: envs.TWITTER_CONSUMER_KEY,
             consumer_secret: envs.TWITTER_CONSUMER_SECRET,
             tokens: twitter_tokens,
+            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
+            // 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
         },
         youtube: {
             key: envs.YOUTUBE_KEY,
@@ -193,6 +197,11 @@ const calculateValue = () => {
         wordpress: {
             cdnUrl: envs.WORDPRESS_CDN,
         },
+        ehentai: {
+            ipb_member_id: envs.EH_IPB_MEMBER_ID,
+            ipb_pass_hash: envs.EH_IPB_PASS_HASH,
+            sk: envs.EH_SK,
+        },
     };
 };
 calculateValue();

package/lib/core_router.js

@@ -0,0 +1,17 @@
+const Router = require('@koa/router');
+const config = require('@/config').value;
+const router = new Router();
+
+// Load Core Route
+router.get('/', require('./routes/index'));
+
+router.get('/robots.txt', (ctx) => {
+    if (config.disallowRobot) {
+        ctx.set('Content-Type', 'text/plain');
+        ctx.body = 'User-agent: *\nDisallow: /';
+    } else {
+        ctx.throw(404, 'Not Found');
+    }
+});
+
+module.exports = router;

package/lib/errors/RequestInProgress.js

@@ -0,0 +1,3 @@
+class RequestInProgressError extends Error {}
+
+module.exports = RequestInProgressError;

package/lib/errors/index.js

@@ -0,0 +1,3 @@
+module.exports = {
+    RequestInProgressError: require('./RequestInProgress'),
+};

package/lib/maintainer.js

@@ -0,0 +1,27 @@
+const dirname = __dirname + '/v2';
+
+// 遍历整个 routes 文件夹，收集模块 maintainer.js
+const maintainerPath = require('require-all')({
+    dirname,
+    filter: /maintainer\.js$/,
+});
+
+const maintainers = {};
+
+// 将收集到的自定义模块进行合并
+for (const dir in maintainerPath) {
+    const routes = maintainerPath[dir]['maintainer.js']; // Do not merge other file
+    for (const key in routes) {
+        maintainers['/' + dir + key] = routes[key];
+    }
+}
+
+// 兼容旧版路由
+const router = require('./router');
+router.stack.forEach((e) => {
+    if (!maintainers[e.path]) {
+        maintainers[e.path] = [];
+    }
+});
+
+module.exports = maintainers;

package/lib/middleware/access-control.js

@@ -29,24 +29,24 @@ module.exports = async (ctx, next) => {
         await next();
     } else {
         if (!isControlled || allowLocalhost) {
-            return await grant();
+            return grant();
         }
 
         if (config.accessKey) {
             if (config.accessKey === accessKey || accessCode === md5(requestPath + config.accessKey)) {
-                return await grant();
+                return grant();
             }
         }
 
         if (config.whitelist) {
             if (config.whitelist.find((white) => ip.includes(white) || requestPath.includes(white) || requestUA.includes(white))) {
-                return await grant();
+                return grant();
             }
         }
 
         if (config.blacklist) {
             if (!config.blacklist.find((black) => ip.includes(black) || requestPath.includes(black) || requestUA.includes(black))) {
-                return await grant();
+                return grant();
             }
         }
 

package/lib/middleware/cache/index.js

@@ -0,0 +1,129 @@
+const md5 = require('@/utils/md5');
+const config = require('@/config').value;
+const logger = require('@/utils/logger');
+const { RequestInProgressError } = require('@/errors');
+
+const globalCache = {
+    get: () => null,
+    set: () => null,
+};
+
+let cacheModule = {
+    get: () => null,
+    set: () => null,
+    status: { available: false },
+    clients: {},
+};
+
+if (config.cache.type === 'redis') {
+    cacheModule = require('./redis');
+    const { redisClient } = cacheModule.clients;
+    globalCache.get = async (key) => {
+        if (key && cacheModule.status.available) {
+            const value = await redisClient.get(key);
+            return value;
+        }
+    };
+    globalCache.set = cacheModule.set;
+} else if (config.cache.type === 'memory') {
+    cacheModule = require('./memory');
+    const { pageCache } = cacheModule.clients;
+    globalCache.get = (key) => {
+        if (key && cacheModule.status.available) {
+            return pageCache.get(key);
+        }
+    };
+    globalCache.set = (key, value, maxAge) => {
+        if (!value || value === 'undefined') {
+            value = '';
+        }
+        if (typeof value === 'object') {
+            value = JSON.stringify(value);
+        }
+        if (key) {
+            return pageCache.set(key, value, maxAge * 1000);
+        }
+    };
+} else {
+    logger.error('Cache not available, concurrent requests are not limited. This could lead to bad behavior.');
+}
+
+// only give cache string, as the `!` condition tricky
+// md5 is used to shrink key size
+// plz, write these tips in comments!
+module.exports = function (app) {
+    const { get, set, status } = cacheModule;
+    app.context.cache = {
+        ...cacheModule,
+        tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire) => {
+            let v = await get(key);
+            if (!v) {
+                v = await getValueFunc();
+                set(key, v, maxAge);
+            } else {
+                let parsed;
+                try {
+                    parsed = JSON.parse(v);
+                } catch (e) {
+                    parsed = null;
+                }
+                if (parsed) {
+                    v = parsed;
+                }
+            }
+
+            return v;
+        },
+        globalCache,
+    };
+
+    return async (ctx, next) => {
+        const key = 'koa-redis-cache:' + md5(ctx.request.path);
+        const controlKey = 'path-requested:' + md5(ctx.request.path);
+
+        if (!status.available) {
+            return next();
+        }
+
+        const isRequesting = await globalCache.get(controlKey);
+
+        if (isRequesting === '1') {
+            throw new RequestInProgressError('This path is currently fetching, please come back later!');
+        }
+
+        try {
+            const value = await globalCache.get(key);
+
+            if (value) {
+                ctx.response.status = 200;
+                if (config.cache.type === 'redis') {
+                    ctx.response.set({
+                        'X-Koa-Redis-Cache': 'true',
+                    });
+                } else if (config.cache.type === 'memory') {
+                    ctx.response.set({
+                        'X-Koa-Memory-Cache': 'true',
+                    });
+                }
+                ctx.state.data = JSON.parse(value);
+                return;
+            }
+        } catch (e) {
+            //
+        }
+
+        // Doesn't hit the cache? We need to let others know!
+        await globalCache.set(controlKey, '1', config.cache.requestTimeout);
+        await next();
+
+        if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) {
+            ctx.state.data.lastBuildDate = new Date().toUTCString();
+            const body = JSON.stringify(ctx.state.data);
+            await globalCache.set(key, body, config.cache.routeExpire);
+        }
+
+        // We need to let it go, even no cache set.
+        // Wait to set cache so the next request could be handled correctly
+        await globalCache.set(controlKey, '0', config.cache.requestTimeout);
+    };
+};

package/lib/middleware/cache/memory.js

@@ -0,0 +1,42 @@
+const Lru = require('lru-cache');
+const config = require('@/config').value;
+
+const status = { available: false };
+
+const pageCache = new Lru({
+    maxAge: config.cache.routeExpire * 1000,
+    max: Infinity,
+});
+
+const routeCache = new Lru({
+    maxAge: config.cache.routeExpire * 1000,
+    max: Infinity,
+    updateAgeOnGet: true,
+});
+
+status.available = true;
+
+module.exports = {
+    get: (key, refresh = true) => {
+        if (key && status.available) {
+            let value = (refresh ? routeCache : pageCache).get(key);
+            if (value) {
+                value = value + '';
+            }
+            return value;
+        }
+    },
+    set: (key, value, maxAge = config.cache.contentExpire, refresh = true) => {
+        if (!value || value === 'undefined') {
+            value = '';
+        }
+        if (typeof value === 'object') {
+            value = JSON.stringify(value);
+        }
+        if (key && status.available) {
+            return (refresh ? routeCache : pageCache).set(key, value, maxAge * 1000);
+        }
+    },
+    clients: { pageCache, routeCache },
+    status,
+};

package/lib/middleware/cache/redis.js

@@ -0,0 +1,48 @@
+const config = require('@/config').value;
+const Redis = require('ioredis');
+const logger = require('@/utils/logger');
+
+const redisClient = new Redis(config.redis.url);
+
+const status = { available: false };
+
+redisClient.on('error', (error) => {
+    status.available = false;
+    logger.error('Redis error: ', error);
+});
+redisClient.on('end', () => {
+    status.available = false;
+});
+redisClient.on('connect', () => {
+    status.available = true;
+    logger.info('Redis connected.');
+});
+
+module.exports = {
+    get: async (key, refresh = true) => {
+        if (key && status.available) {
+            let value = await redisClient.get(key);
+            if (value && refresh) {
+                redisClient.expire(key, config.cache.contentExpire);
+                value = value + '';
+            }
+            return value;
+        }
+    },
+    set: (key, value, maxAge = config.cache.contentExpire) => {
+        if (!status.available) {
+            return;
+        }
+        if (!value || value === 'undefined') {
+            value = '';
+        }
+        if (typeof value === 'object') {
+            value = JSON.stringify(value);
+        }
+        if (key) {
+            return redisClient.set(key, value, 'EX', maxAge); // setMode: https://redis.io/commands/set
+        }
+    },
+    clients: { redisClient },
+    status,
+};

package/lib/middleware/header.js

@@ -15,7 +15,7 @@ module.exports = async (ctx, next) => {
     logger.info(`${ctx.url}, user IP: ${ctx.ips[0] || ctx.ip}`);
     ctx.set(headers);
     ctx.set({
-        'Access-Control-Allow-Origin': `${config.allowOrigin || ctx.host}`,
+        'Access-Control-Allow-Origin': config.allowOrigin || ctx.host,
     });
 
     await next();

package/lib/middleware/load-on-demand.js

@@ -0,0 +1,36 @@
+const mount = require('koa-mount');
+const Router = require('@koa/router');
+const routes = require('../v2router');
+const loadedRoutes = new Set();
+
+module.exports = function (app) {
+    return async function (ctx, next) {
+        const p = ctx.request.path.split('/').filter(Boolean);
+        let modName = null;
+        let mounted = false;
+
+        if (p.length > 0) {
+            modName = p[0];
+            if (!loadedRoutes.has(modName)) {
+                const mod = routes[modName];
+                // Mount module
+                if (mod) {
+                    mounted = true;
+                    loadedRoutes.add(modName);
+                    const router = new Router();
+                    mod(router);
+                    app.use(mount(`/${modName}`, router.routes())).use(router.allowedMethods());
+                }
+            } else {
+                mounted = true;
+            }
+        }
+
+        await next();
+
+        // We should only add it when koa router matched
+        if (mounted && ctx._matchedRoute) {
+            ctx._matchedRoute = `/${modName}${ctx._matchedRoute}`;
+        }
+    };
+};

package/lib/middleware/onerror.js

@@ -1,10 +1,14 @@
 const logger = require('@/utils/logger');
 const config = require('@/config').value;
-const Sentry = require('@sentry/node');
 const art = require('art-template');
 const path = require('path');
 
+const { RequestInProgressError } = require('@/errors');
+
+let Sentry;
+
 if (config.sentry.dsn) {
+    Sentry = Sentry || require('@sentry/node');
     Sentry.init({
         dsn: config.sentry.dsn,
     });
@@ -31,77 +35,10 @@ module.exports = async (ctx, next) => {
         let message = err;
         if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
             message = `${err.message}: target website might be blocking our access, you can <a href="https://docs.rsshub.app/install/">host your own RSSHub instance</a> for a better usability.`;
-
-            const templateZh = `<!--
-请确保已阅读 [文档](https://docs.rsshub.app) 内相关部分，并按照模版提供信息，否则 issue 将被立即关闭。
-
-由于部分源网站反爬缘故，演示地址一些 rss 会返回 status code 403，该问题不是 RSSHub 所致，请勿提交 issue。
--->
-
-- 完整路由地址，包含所有必选与可选参数
-    \`${ctx.path}\`
-- 预期是什么
-
-- 实际发生了什么
-
-- 部署相关信息
-
-<!--
-如果是演示地址(rsshub.app)有此问题请删除此部分。
-
-请确保您部署的是[主线 master 分支](https://github.com/DIYgod/RSSHub/tree/master)最新版 RSSHub。
--->
-
-| Env                | Value         |
-| ------------------ | ------------- |
-| OS                 |               |
-| Node version       |${process.version}     |
-| if Docker, version |               |
-
-- 额外信息（日志、报错等）
-\`\`\`
-    ${err.message}
-\`\`\` `;
-
-            const templateEn = `<!--
-Please ensure you have read [documentation](https://docs.rsshub.app/en), and provide all the information required by this template, otherwise the issue will be closed immediately.
-
-Due to the anti-crawling policy implemented by certain websites, some RSS routes provided by the demo will return status code 403. This is not an issue caused by RSSHub and please do not report it.
--->
-
-- The involved route, with all required and optional parameters
-    \`${ctx.path}\`
-- What is expected
-
-- What is actually happening
-
-- Self-deployment information
-
-<!--
-Delete this section if you are using [RSSHub demo](https://rsshub.app).
-
-Please ensure you have deployed the [master branch](https://github.com/DIYgod/RSSHub/tree/master) of RSSHub.
--->
-
-| Env                | Value         |
-| ------------------ | ------------- |
-| OS                 |               |
-| Node version       |${process.version}     |
-| if Docker, version |               |
-
-- Additional info (logs errors etc)
-\`\`\`
-    ${err.message}
-\`\`\` `;
-
-            message += `<br><br>如果您认为 RSSHub 导致了该错误，<a href="https://github.com/DIYgod/RSSHub/issues/new?template=bug_report_zh.md&title=[BUG]&body=${encodeURI(templateZh)}" target="_blank">请在 GitHub 报告</a>。`;
-
-            message += `<br><br>If you believe this is an error caused by RSSHub, please <a href="https://github.com/DIYgod/RSSHub/issues/new?template=bug_report_en.md&title=[BUG]&body=${encodeURI(
-                templateEn
-            )}" target="_blank">report me on GitHub</a>.`;
         } else if (err instanceof Error) {
             message = process.env.NODE_ENV === 'production' ? err.message : err.stack;
         }
+
         logger.error(`Error in ${ctx.request.path}: ${message}`);
 
         if (config.isPackage) {
@@ -114,7 +51,11 @@ Please ensure you have deployed the [master branch](https://github.com/DIYgod/RS
             ctx.set({
                 'Content-Type': 'text/html; charset=UTF-8',
             });
-            if (ctx.status === 403) {
+
+            if (err instanceof RequestInProgressError) {
+                ctx.status = 503;
+                message = err.message;
+            } else if (ctx.status === 403) {
                 message = err.message;
             } else {
                 ctx.status = 404;
@@ -125,6 +66,8 @@ Please ensure you have deployed the [master branch](https://github.com/DIYgod/RS
             ctx.body = art(path.resolve(__dirname, '../views/error.art'), {
                 requestPath,
                 message,
+                errorPath: ctx.path,
+                nodeVersion: process.version,
             });
         }
 

package/lib/middleware/parameter.js

@@ -1,10 +1,11 @@
 const entities = require('entities');
-const mercury_parser = require('@postlight/mercury-parser');
 const cheerio = require('cheerio');
 const { simplecc } = require('simplecc-wasm');
 const got = require('@/utils/got');
 const config = require('@/config').value;
 
+let mercury_parser;
+
 module.exports = async (ctx, next) => {
     await next();
 
@@ -30,7 +31,7 @@ module.exports = async (ctx, next) => {
         // sort items
         ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
 
-        const handleItem = async (item) => {
+        const handleItem = (item) => {
             item.title && (item.title = entities.decodeXML(item.title + ''));
 
             // handle pubDate
@@ -204,6 +205,8 @@ module.exports = async (ctx, next) => {
                     const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
                         // if parser failed, return default description and not report error
                         try {
+                            mercury_parser = mercury_parser || require('@postlight/mercury-parser');
+
                             const res = await got(link);
                             const $ = cheerio.load(res.data);
                             const result = await mercury_parser.parse(link, {
@@ -236,6 +239,29 @@ module.exports = async (ctx, next) => {
                     item.description = simplecc(item.description, ctx.query.opencc);
                 });
             }
+
+            // brief
+            if (ctx.query.brief) {
+                const num = /[1-9]\d{2,}/;
+                if (num.test(ctx.query.brief)) {
+                    ctx.query.brief = parseInt(ctx.query.brief);
+                    ctx.state.data.item.forEach((item) => {
+                        let text;
+                        if (item.description) {
+                            text = item.description.replace(/<\/?[^>]+(>|$)/g, '');
+                        }
+                        if (text && text.length) {
+                            if (text.length > ctx.query.brief) {
+                                item.description = `<p>${text.substring(0, ctx.query.brief)}…</p>`;
+                            } else {
+                                item.description = `<p>${text}</p>`;
+                            }
+                        }
+                    });
+                } else {
+                    throw Error(`Invalid parameter <code>brief=${ctx.query.brief}</code>. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
+                }
+            }
         }
     }
 };

package/lib/middleware/template.js

@@ -1,14 +1,9 @@
 const art = require('art-template');
 const path = require('path');
 const config = require('@/config').value;
-const typeRegex = /\.(atom|rss)$/;
-const unsupportedRegex = /\.json$/;
+const typeRegex = /\.(atom|rss|debug\.json)$/;
 
 module.exports = async (ctx, next) => {
-    if (ctx.request.path.match(unsupportedRegex)) {
-        throw Error('<b>JSON output had been removed, see: <a href="https://github.com/DIYgod/RSSHub/issues/1114">https://github.com/DIYgod/RSSHub/issues/1114</a></b>');
-    }
-
     if (ctx.headers['user-agent'] && ctx.headers['user-agent'].includes('Reeder')) {
         ctx.request.path = ctx.request.path.replace(/.com$/, '');
     }
@@ -18,6 +13,17 @@ module.exports = async (ctx, next) => {
 
     await next();
 
+    if (ctx.state.type[1] === 'debug.json' && config.debugInfo) {
+        ctx.set({
+            'Content-Type': 'application/json; charset=UTF-8',
+        });
+        if (ctx.state.json) {
+            ctx.body = JSON.stringify(ctx.state.json, null, 4);
+        } else {
+            ctx.body = JSON.stringify({ message: 'plugin does not set json' });
+        }
+    }
+
     if (!ctx.body) {
         let template;