rsshub 1.0.0-master.f6ab7b6 → 1.0.0-master.f6b1d4c

package/README.md

@@ -1,5 +1,5 @@
 <p align="center">
-<img src="https://i.loli.net/2019/04/23/5cbeb7e41414c.png" alt="RSSHub" width="100">
+<img src="https://docs.rsshub.app/logo.png" alt="RSSHub" width="100">
 </p>
 <h1 align="center">RSSHub</h1>
 
@@ -10,7 +10,8 @@
 [![docker publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/docker-release.yml?branch=master&label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
 [![test](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/test.yml?branch=master&label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
 [![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square&logo=codecov)](https://app.codecov.io/gh/DIYgod/RSSHub/branch/master)
-[![Foresight Docs](https://api-public.service.runforesight.com/api/v1/badge/success?repoId=5da20e56-cecf-4b05-9bcd-cab097fb2dca)](https://docs.runforesight.com/)
+[![CodeFactor](https://www.codefactor.io/repository/github/diygod/rsshub/badge)](https://www.codefactor.io/repository/github/diygod/rsshub)
+[![DeepScan grade](https://deepscan.io/api/teams/6244/projects/8135/branches/92448/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=6244&pid=8135&bid=92448)
 
 ## Introduction
 
@@ -32,26 +33,6 @@ RSSHub 是一个开源、简单易用、易于扩展的 RSS 生成器，可以
 
 ## Special Thanks
 
-### Special Sponsors
-
-<p>
-<a href="https://rss3.io/" target="_blank"><img height="60px" src="https://i.imgur.com/vHtI4bz.png"></a>
-</p>
-
-<table width="100%">
-<tr>
-<td><img width="1000" height="0"><a href="https://www.runforesight.com/?utm_source=RSSHub&utm_medium=sponsorship"><img src="https://www.thundra.io/hubfs/RunForesight/GitHub%20Sponsorship%20banners/github-actions-observability.png"
-alt="Foresight" width="260" align="right"></a><h3>Foresight: Increase CI/CD Health & Test Performance</h3><p>
-Foresight provides full visibility and deep insights into the health and performance of your tests and CI pipelines. Assess the risk of code changes, deal with flaky tests, see workflow and test trends over time, and improve the contributing experience with Foresight.</p>
-
-[Sign up now!](https://runforesight.com/?utm_source=RSSHub&utm_medium=sponsorship)
-
-</td>
-</tr>
-</table>
-
-[![](https://opencollective.com/static/images/become_sponsor.svg)](https://docs.rsshub.app/en/support/)
-
 ### Contributors
 
 [![](https://opencollective.com/RSSHub/contributors.svg?width=890)](https://github.com/DIYgod/RSSHub/graphs/contributors)
@@ -98,6 +79,7 @@ You can support RSSHub via donations.
 Recurring donors will be rewarded via express issue response, or even have your name displayed on our GitHub page and website.
 
 -   Become a Sponser on [GitHub](https://github.com/sponsors/DIYgod)
+-   Become a Sponser on [Open Collective](https://opencollective.com/RSSHub)
 -   Become a Sponser on [Patreon](https://www.patreon.com/DIYgod)
 -   Become a Sponser on [爱发电](https://afdian.net/@diygod)
 -   Contact us directly: i@diygod.me
@@ -106,8 +88,8 @@ Recurring donors will be rewarded via express issue response, or even have your
 
 We accept donations via the following ways:
 
--   [WeChat Pay](https://diygod.me/images/wx.jpg)
--   [Alipay](https://diygod.me/images/zfb.jpg)
+-   [WeChat Pay](https://archive.diygod.me/images/wx.jpg)
+-   [Alipay](https://archive.diygod.me/images/zfb.jpg)
 -   [Paypal](https://www.paypal.me/DIYgod)
 
 ## Author

package/lib/api_router.js

@@ -1,11 +1,11 @@
 const Router = require('@koa/router');
 const router = new Router();
-const maintainer = require('./maintainer');
 
 router.get('/routes/:name?', (ctx) => {
     const result = {};
     let counter = 0;
 
+    const maintainer = require('./maintainer');
     Object.keys(maintainer).forEach((i) => {
         const path = i;
         const top = path.split('/')[1];

package/lib/app.js

@@ -13,6 +13,7 @@ const cache = require('./middleware/cache');
 const parameter = require('./middleware/parameter');
 const template = require('./middleware/template');
 const favicon = require('koa-favicon');
+const serve = require('koa-static');
 const debug = require('./middleware/debug');
 const accessControl = require('./middleware/access-control');
 const antiHotlink = require('./middleware/anti-hotlink');
@@ -37,6 +38,7 @@ app.proxy = true;
 
 // favicon
 app.use(favicon(__dirname + '/favicon.png'));
+app.use(serve(__dirname + '/static'));
 
 // global error handing
 app.use(onerror);

package/lib/config.js

@@ -9,6 +9,7 @@ const calculateValue = () => {
     const twitter_tokens = {};
     const email_config = {};
     const discuz_cookies = {};
+    const medium_cookies = {};
 
     for (const name in envs) {
         if (name.startsWith('BILIBILI_COOKIE_')) {
@@ -23,6 +24,9 @@ const calculateValue = () => {
         } else if (name.startsWith('DISCUZ_COOKIE_')) {
             const cid = name.slice(14);
             discuz_cookies[cid] = envs[name];
+        } else if (name.startsWith('MEDIUM_COOKIE_')) {
+            const username = name.slice(14).toLowerCase();
+            medium_cookies[username] = envs[name];
         }
     }
 
@@ -97,6 +101,7 @@ const calculateValue = () => {
         feature: {
             allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
             filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
+            allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true',
         },
         suffix: envs.SUFFIX,
         titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
@@ -119,10 +124,16 @@ const calculateValue = () => {
         chuiniu: {
             member: envs.CHUINIU_MEMBER,
         },
+        civitai: {
+            cookie: envs.CIVITAI_COOKIE,
+        },
         dida365: {
             username: envs.DIDA365_USERNAME,
             password: envs.DIDA365_PASSWORD,
         },
+        discord: {
+            authorization: envs.DISCORD_AUTHORIZATION,
+        },
         discuz: {
             cookies: discuz_cookies,
         },
@@ -184,9 +195,11 @@ const calculateValue = () => {
             username: envs.IG_USERNAME,
             password: envs.IG_PASSWORD,
             proxy: envs.IG_PROXY,
+            cookie: envs.IG_COOKIE,
         },
         iwara: {
-            cookie: envs.IWARA_COOKIE,
+            username: envs.IWARA_USERNAME,
+            password: envs.IWARA_PASSWORD,
         },
         lastfm: {
             api_key: envs.LASTFM_API_KEY,
@@ -199,6 +212,10 @@ const calculateValue = () => {
             accessToken: envs.MASTODON_API_ACCESS_TOKEN,
             acctDomain: envs.MASTODON_API_ACCT_DOMAIN,
         },
+        medium: {
+            cookies: medium_cookies,
+            articleCookie: envs.MEDIUM_ARTICLE_COOKIE || '',
+        },
         miniflux: {
             instance: envs.MINIFLUX_INSTANCE || 'https://reader.miniflux.app',
             token: envs.MINIFLUX_TOKEN || '',
@@ -217,15 +234,21 @@ const calculateValue = () => {
             username: envs.NHENTAI_USERNAME,
             password: envs.NHENTAI_PASSWORD,
         },
+        notion: {
+            key: envs.NOTION_TOKEN,
+        },
         pianyuan: {
             cookie: envs.PIANYUAN_COOKIE,
         },
+        pixabay: {
+            key: envs.PIXABAY_KEY,
+        },
         pixiv: {
             refreshToken: envs.PIXIV_REFRESHTOKEN,
             bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
             bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
             bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
-            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
+            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.re',
         },
         pkubbs: {
             cookie: envs.PKUBBS_COOKIE,
@@ -233,6 +256,9 @@ const calculateValue = () => {
         saraba1st: {
             cookie: envs.SARABA1ST_COOKIE,
         },
+        sehuatang: {
+            cookie: envs.SEHUATANG_COOKIE,
+        },
         scboy: {
             token: envs.SCBOY_BBS_TOKEN,
         },
@@ -257,6 +283,7 @@ const calculateValue = () => {
         weibo: {
             app_key: envs.WEIBO_APP_KEY,
             app_secret: envs.WEIBO_APP_SECRET,
+            cookies: envs.WEIBO_COOKIES,
             redirect_url: envs.WEIBO_REDIRECT_URL,
         },
         wenku8: {
@@ -278,9 +305,6 @@ const calculateValue = () => {
             clientSecret: envs.YOUTUBE_CLIENT_SECRET,
             refreshToken: envs.YOUTUBE_REFRESH_TOKEN,
         },
-        yuque: {
-            token: envs.YUQUE_TOKEN,
-        },
         zhihu: {
             cookies: envs.ZHIHU_COOKIES,
         },

package/lib/maintainer.js

@@ -1,4 +1,14 @@
 const dirname = __dirname + '/v2';
+const fs = require('fs');
+const { join } = require('path');
+
+// Presence Check
+for (const dir of fs.readdirSync(dirname)) {
+    const dirPath = join(dirname, dir);
+    if (!fs.existsSync(join(dirPath, 'maintainer.js'))) {
+        throw Error(`No maintainer.js in "${dirPath}".`);
+    }
+}
 
 // 遍历整个 routes 文件夹，收集模块 maintainer.js
 const maintainerPath = require('require-all')({
@@ -11,6 +21,21 @@ const maintainers = {};
 // 将收集到的自定义模块进行合并
 for (const dir in maintainerPath) {
     const routes = maintainerPath[dir]['maintainer.js']; // Do not merge other file
+
+    // typo check e.g., ✘ module.export, ✔ module.exports
+    if (!Object.keys(routes).length) {
+        throw Error(`No maintainer in "${dir}".`);
+    }
+    for (const author of Object.values(routes)) {
+        if (!Array.isArray(author)) {
+            throw Error(`Maintainers' name should be an array in "${dir}".`);
+        }
+        // check for [], [''] or ['Someone', '']
+        if (author.length < 1 || author.includes('')) {
+            throw Error(`Empty maintainer in "${dir}".`);
+        }
+    }
+
     for (const key in routes) {
         maintainers['/' + dir + key] = routes[key];
     }

package/lib/middleware/cache/memory.js

@@ -1,9 +1,9 @@
-const Lru = require('lru-cache');
+const { LRUCache } = require('lru-cache');
 const config = require('@/config').value;
 
 const status = { available: false };
 
-const memoryCache = new Lru({
+const memoryCache = new LRUCache({
     ttl: config.cache.routeExpire * 1000,
     max: config.memory.max,
 });

package/lib/middleware/parameter.js

@@ -139,6 +139,13 @@ module.exports = async (ctx, next) => {
                 });
 
                 item.description = $('body').html() + '' + (config.suffix || '');
+
+                if (item._extra?.links && $('.rsshub-quote').length) {
+                    item._extra?.links?.map((e) => {
+                        e.content_html = $.html($('.rsshub-quote'));
+                        return e;
+                    });
+                }
             }
             return item;
         };
@@ -266,7 +273,7 @@ module.exports = async (ctx, next) => {
                     });
 
                     item.author = author || (parsed_result ? parsed_result.author : '');
-                    item.description = parsed_result ? entities.decodeXML(parsed_result.content) : description;
+                    item.description = parsed_result && parsed_result.content.length > 40 ? entities.decodeXML(parsed_result.content) : description;
                 });
                 await Promise.all(tasks);
             }
@@ -306,7 +313,7 @@ module.exports = async (ctx, next) => {
                         }
                     });
                 } else {
-                    throw Error(`Invalid parameter <code>brief=${ctx.query.brief}</code>. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
+                    throw Error(`Invalid parameter brief. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
                 }
             }
 

package/lib/middleware/template.js

@@ -1,7 +1,7 @@
-const art = require('art-template');
+const { art, json } = require('@/utils/render');
 const path = require('path');
 const config = require('@/config').value;
-const typeRegex = /\.(atom|rss|debug\.json)$/;
+const typeRegex = /\.(atom|rss|debug\.json|json|\d+\.debug\.html)$/;
 const { collapseWhitespace, convertDateToISO8601 } = require('@/utils/common-utils');
 
 module.exports = async (ctx, next) => {
@@ -14,21 +14,38 @@ module.exports = async (ctx, next) => {
 
     await next();
 
-    if (ctx.state.type[1] === 'debug.json' && config.debugInfo) {
+    const outputType = ctx.state.type[1] || 'rss';
+
+    if (outputType === 'debug.json' && config.debugInfo) {
         ctx.set({
             'Content-Type': 'application/json; charset=UTF-8',
         });
         if (ctx.state.json) {
             ctx.body = JSON.stringify(ctx.state.json, null, 4);
         } else {
-            ctx.body = JSON.stringify({ message: 'plugin does not set json' });
+            ctx.body = JSON.stringify({ message: 'plugin does not set debug json' });
+        }
+    }
+    if (outputType.endsWith('.debug.html') && config.debugInfo) {
+        ctx.set({
+            'Content-Type': 'text/html; charset=UTF-8',
+        });
+
+        const index = parseInt(outputType.match(/(\d+)\.debug\.html$/)[1]);
+        if (!(ctx.state.data && ctx.state.data.item && ctx.state.data.item[index])) {
+            ctx.body = `ctx.state.data.item[${index}] not found`;
+        } else {
+            ctx.body = ctx.state.data.item[index].description;
         }
     }
 
+    if (outputType === 'json') {
+        ctx.set({ 'Content-Type': 'application/feed+json; charset=UTF-8' });
+    }
+
     if (!ctx.body) {
         let template;
 
-        const outputType = ctx.state.type[1];
         switch (outputType) {
             case 'atom':
                 template = path.resolve(__dirname, '../views/atom.art');
@@ -60,7 +77,16 @@ module.exports = async (ctx, next) => {
                         }
                     }
 
-                    item.author = collapseWhitespace(item.author);
+                    if (typeof item.author === 'string') {
+                        item.author = collapseWhitespace(item.author);
+                    } else if (typeof item.author === 'object' && item.author !== null) {
+                        for (const a of item.author) {
+                            a.name = collapseWhitespace(a.name);
+                        }
+                        if (outputType !== 'json') {
+                            item.author = item.author.map((a) => a.name).join(', ');
+                        }
+                    }
 
                     if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && item.itunes_duration.indexOf(':') === -1) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) {
                         item.itunes_duration = +item.itunes_duration;
@@ -68,7 +94,7 @@ module.exports = async (ctx, next) => {
                             Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2);
                     }
 
-                    if (outputType === 'atom') {
+                    if (outputType !== 'rss') {
                         item.pubDate = convertDateToISO8601(item.pubDate);
                         item.updated = convertDateToISO8601(item.updated);
                     }
@@ -86,10 +112,15 @@ module.exports = async (ctx, next) => {
         };
         if (config.isPackage) {
             ctx.body = data;
-        } else {
-            if (template) {
-                ctx.body = art(template, data);
-            }
+            return;
+        }
+        if (!template) {
+            return;
+        }
+        if (outputType !== 'json') {
+            ctx.body = art(template, data);
+            return;
         }
+        ctx.body = json(data);
     }
 };