rsshub 1.0.0-master.fdefa01 → 1.0.0-master.ff1cc11

package/lib/middleware/onerror.js

@@ -1,12 +1,16 @@
 const logger = require('@/utils/logger');
 const config = require('@/config').value;
-const Sentry = require('@sentry/node');
 const art = require('art-template');
 const path = require('path');
 
-if (config.sentry) {
+const { RequestInProgressError } = require('@/errors');
+
+let Sentry;
+
+if (config.sentry.dsn) {
+    Sentry = Sentry || require('@sentry/node');
     Sentry.init({
-        dsn: config.sentry,
+        dsn: config.sentry.dsn,
     });
     Sentry.configureScope((scope) => {
         scope.setTag('node_name', config.nodeName);
@@ -17,95 +21,41 @@ if (config.sentry) {
 
 module.exports = async (ctx, next) => {
     try {
+        const time = +new Date();
         await next();
+        if (config.sentry.dsn && +new Date() - time >= config.sentry.routeTimeout) {
+            Sentry.withScope((scope) => {
+                scope.setTag('route', ctx._matchedRoute);
+                scope.setTag('name', ctx.request.path.split('/')[1]);
+                scope.addEventProcessor((event) => Sentry.Handlers.parseRequest(event, ctx.request));
+                Sentry.captureException(new Error('Route Timeout'));
+            });
+        }
     } catch (err) {
         let message = err;
         if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
             message = `${err.message}: target website might be blocking our access, you can <a href="https://docs.rsshub.app/install/">host your own RSSHub instance</a> for a better usability.`;
-
-            const templateZh = `<!--
-请确保已阅读 [文档](https://docs.rsshub.app) 内相关部分，并按照模版提供信息，否则 issue 将被立即关闭。
-
-由于部分源网站反爬缘故，演示地址一些 rss 会返回 status code 403，该问题不是 RSSHub 所致，请勿提交 issue。
--->
-
-- 完整路由地址，包含所有必选与可选参数
-    \`${ctx.path}\`
-- 预期是什么
-
-- 实际发生了什么
-
-- 部署相关信息
-
-<!--
-如果是演示地址(rsshub.app)有此问题请删除此部分。
-
-请确保您部署的是[主线 master 分支](https://github.com/DIYgod/RSSHub/tree/master)最新版 RSSHub。
--->
-
-| Env                | Value         |
-| ------------------ | ------------- |
-| OS                 |               |
-| Node version       |${process.version}     |
-| if Docker, version |               |
-
-- 额外信息（日志、报错等）
-\`\`\`
-    ${err.message}
-\`\`\` `;
-
-            const templateEn = `<!--
-Please ensure you have read [documentation](https://docs.rsshub.app/en), and provide all the information required by this template, otherwise the issue will be closed immediately.
-
-Due to the anti-crawling policy implemented by certain websites, some RSS routes provided by the demo will return status code 403. This is not an issue caused by RSSHub and please do not report it.
--->
-
-- The involved route, with all required and optional parameters
-    \`${ctx.path}\`
-- What is expected
-
-- What is actually happening
-
-- Self-deployment information
-
-<!--
-Delete this section if you are using [RSSHub demo](https://rsshub.app).
-
-Please ensure you have deployed the [master branch](https://github.com/DIYgod/RSSHub/tree/master) of RSSHub.
--->
-
-| Env                | Value         |
-| ------------------ | ------------- |
-| OS                 |               |
-| Node version       |${process.version}     |
-| if Docker, version |               |
-
-- Additional info (logs errors etc)
-\`\`\`
-    ${err.message}
-\`\`\` `;
-
-            message += `<br><br>如果您认为 RSSHub 导致了该错误，<a href="https://github.com/DIYgod/RSSHub/issues/new?template=bug_report_zh.md&title=[BUG]&body=${encodeURI(templateZh)}" target="_blank">请在 GitHub 报告</a>。`;
-
-            message += `<br><br>If you believe this is an error caused by RSSHub, please <a href="https://github.com/DIYgod/RSSHub/issues/new?template=bug_report_en.md&title=[BUG]&body=${encodeURI(
-                templateEn
-            )}" target="_blank">report me on GitHub</a>.`;
         } else if (err instanceof Error) {
             message = process.env.NODE_ENV === 'production' ? err.message : err.stack;
         }
+
         logger.error(`Error in ${ctx.request.path}: ${message}`);
 
         if (config.isPackage) {
             ctx.body = {
                 error: {
-                    message: err.message,
+                    message: err.message ? err.message : err,
                 },
             };
         } else {
             ctx.set({
                 'Content-Type': 'text/html; charset=UTF-8',
             });
-            if (ctx.status === 403) {
+
+            if (err instanceof RequestInProgressError) {
+                ctx.status = 503;
+                message = err.message;
+            } else if (ctx.status === 403) {
                 message = err.message;
             } else {
                 ctx.status = 404;
@@ -116,6 +66,8 @@ Please ensure you have deployed the [master branch](https://github.com/DIYgod/RS
             ctx.body = art(path.resolve(__dirname, '../views/error.art'), {
                 requestPath,
                 message,
+                errorPath: ctx.path,
+                nodeVersion: process.version,
             });
         }
 
@@ -140,7 +92,7 @@ Please ensure you have deployed the [master branch](https://github.com/DIYgod/RS
             }
         }
 
-        if (config.sentry) {
+        if (config.sentry.dsn) {
             Sentry.withScope((scope) => {
                 scope.setTag('route', ctx._matchedRoute);
                 scope.setTag('name', ctx.request.path.split('/')[1]);

package/lib/middleware/parameter.js

@@ -1,10 +1,11 @@
-const he = require('he');
-const mercury_parser = require('@postlight/mercury-parser');
+const entities = require('entities');
 const cheerio = require('cheerio');
-const OpenCC = require('opencc');
+const { simplecc } = require('simplecc-wasm');
 const got = require('@/utils/got');
 const config = require('@/config').value;
 
+let mercury_parser;
+
 module.exports = async (ctx, next) => {
     await next();
 
@@ -24,14 +25,14 @@ module.exports = async (ctx, next) => {
         ctx.state.data.item = ctx.state.data.item || [];
 
         // decode HTML entities
-        ctx.state.data.title && (ctx.state.data.title = he.decode(ctx.state.data.title + ''));
-        ctx.state.data.description && (ctx.state.data.description = he.decode(ctx.state.data.description + ''));
+        ctx.state.data.title && (ctx.state.data.title = entities.decodeXML(ctx.state.data.title + ''));
+        ctx.state.data.description && (ctx.state.data.description = entities.decodeXML(ctx.state.data.description + ''));
 
         // sort items
         ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
 
-        ctx.state.data.item.forEach((item) => {
-            item.title && (item.title = he.decode(item.title + ''));
+        const handleItem = (item) => {
+            item.title && (item.title = entities.decodeXML(item.title + ''));
 
             // handle pubDate
             if (item.pubDate) {
@@ -111,9 +112,12 @@ module.exports = async (ctx, next) => {
                         $ele.removeAttr(e);
                     });
                 });
-                item.description = he.decode($('body').html() + '') + (config.suffix || '');
+                item.description = entities.decodeXML($('body').html() + '') + (config.suffix || '');
             }
-        });
+            return item;
+        };
+
+        ctx.state.data.item = await Promise.all(ctx.state.data.item.map(handleItem));
 
         if (ctx.query) {
             // limit
@@ -201,6 +205,8 @@ module.exports = async (ctx, next) => {
                     const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
                         // if parser failed, return default description and not report error
                         try {
+                            mercury_parser = mercury_parser || require('@postlight/mercury-parser');
+
                             const res = await got(link);
                             const $ = cheerio.load(res.data);
                             const result = await mercury_parser.parse(link, {
@@ -228,12 +234,33 @@ module.exports = async (ctx, next) => {
 
             // opencc
             if (ctx.query.opencc) {
-                const converter = new OpenCC(`${ctx.query.opencc}.json`);
-                const tasks = ctx.state.data.item.map(async (item) => {
-                    item.title = await converter.convertPromise(item.title);
-                    item.description = await converter.convertPromise(item.description);
+                ctx.state.data.item.forEach((item) => {
+                    item.title = simplecc(item.title, ctx.query.opencc);
+                    item.description = simplecc(item.description, ctx.query.opencc);
                 });
-                await Promise.all(tasks);
+            }
+
+            // brief
+            if (ctx.query.brief) {
+                const num = /[1-9]\d{2,}/;
+                if (num.test(ctx.query.brief)) {
+                    ctx.query.brief = parseInt(ctx.query.brief);
+                    ctx.state.data.item.forEach((item) => {
+                        let text;
+                        if (item.description) {
+                            text = item.description.replace(/<\/?[^>]+(>|$)/g, '');
+                        }
+                        if (text && text.length) {
+                            if (text.length > ctx.query.brief) {
+                                item.description = `<p>${text.substring(0, ctx.query.brief)}…</p>`;
+                            } else {
+                                item.description = `<p>${text}</p>`;
+                            }
+                        }
+                    });
+                } else {
+                    throw Error(`Invalid parameter <code>brief=${ctx.query.brief}</code>. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
+                }
             }
         }
     }

package/lib/middleware/template.js

@@ -1,19 +1,29 @@
 const art = require('art-template');
 const path = require('path');
 const config = require('@/config').value;
-const typeRegrx = /\.(atom|rss)$/;
-const unsupportedRegrx = /\.json$/;
+const typeRegex = /\.(atom|rss|debug\.json)$/;
 
 module.exports = async (ctx, next) => {
-    if (ctx.request.path.match(unsupportedRegrx)) {
-        throw Error('<b>JSON output had been removed, see: <a href="https://github.com/DIYgod/RSSHub/issues/1114">https://github.com/DIYgod/RSSHub/issues/1114</a></b>');
+    if (ctx.headers['user-agent'] && ctx.headers['user-agent'].includes('Reeder')) {
+        ctx.request.path = ctx.request.path.replace(/.com$/, '');
     }
 
-    ctx.state.type = ctx.request.path.match(typeRegrx) || ['', ''];
-    ctx.request.path = ctx.request.path.replace(typeRegrx, '');
+    ctx.state.type = ctx.request.path.match(typeRegex) || ['', ''];
+    ctx.request.path = ctx.request.path.replace(typeRegex, '');
 
     await next();
 
+    if (ctx.state.type[1] === 'debug.json' && config.debugInfo) {
+        ctx.set({
+            'Content-Type': 'application/json; charset=UTF-8',
+        });
+        if (ctx.state.json) {
+            ctx.body = JSON.stringify(ctx.state.json, null, 4);
+        } else {
+            ctx.body = JSON.stringify({ message: 'plugin does not set json' });
+        }
+    }
+
     if (!ctx.body) {
         let template;
 
@@ -44,10 +54,6 @@ module.exports = async (ctx, next) => {
                         }
                     }
 
-                    if (!item.pubDate) {
-                        item.pubDate = new Date().toISOString();
-                    }
-
                     if (item.enclosure_length) {
                         const itunes_duration =
                             Math.floor(item.enclosure_length / 3600) + ':' + (Math.floor((item.enclosure_length % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.enclosure_length % 3600) % 60) / 100).toFixed(2).slice(-2);