rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f812e91

package/README.md

@@ -5,14 +5,16 @@
 
 > 🍰 Everything is RSSible
 
-[![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?logo=telegram&style=flat-square)](https://t.me/rsshub)
-[![npm publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/npm-publish.yml?branch=master&label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
-[![docker publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/docker-release.yml?branch=master&label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
+[![](https://img.shields.io/badge/dynamic/json?url=https://rsshub-analytics.diygod.workers.dev/&query=requests&color=F38020&label=requests&logo=cloudflare&style=flat-square&suffix=/month)](https://rsshub.app)
+[![docker publish](https://img.shields.io/docker/pulls/diygod/rsshub?label=docker%20pulls&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
+[![npm publish](https://img.shields.io/npm/dt/rsshub?label=npm%20downloads&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
 [![test](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/test.yml?branch=master&label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
 [![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square&logo=codecov)](https://app.codecov.io/gh/DIYgod/RSSHub/branch/master)
 [![CodeFactor](https://www.codefactor.io/repository/github/diygod/rsshub/badge)](https://www.codefactor.io/repository/github/diygod/rsshub)
 [![DeepScan grade](https://deepscan.io/api/teams/6244/projects/8135/branches/92448/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=6244&pid=8135&bid=92448)
 
+[![Telegram group](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.swo.moe%2Fstats%2Ftelegram%2Frsshub&query=count&color=2CA5E0&label=Telegram%20Group&logo=telegram&cacheSeconds=3600&style=flat-square)](https://t.me/rsshub) [![Telegram channel](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.swo.moe%2Fstats%2Ftelegram%2FawesomeRSSHub&query=count&color=2CA5E0&label=Telegram%20Channel&logo=telegram&cacheSeconds=3600&style=flat-square)](https://t.me/awesomeRSSHub)
+
 ## Introduction
 
 RSSHub is an open source, easy to use, and extensible RSS feed generator. It's capable of generating RSS feeds from pretty much everything.
@@ -30,7 +32,7 @@ RSSHub can be used with browser extension [RSSHub Radar](https://github.com/DIYg
 ### Special Sponsors
 
 <p>
-<a href="https://xlog.app/" target="_blank"><img height="100px" src="https://i.imgur.com/JuhHTKD.png"></a>
+<a href="https://rss3.io" target="_blank"><img height="50px" src="https://i.imgur.com/lb1dDGK.png"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="https://xlog.app/" target="_blank"><img height="50px" src="https://i.imgur.com/JuhHTKD.png"></a>
 </p>
 
 [![](https://opencollective.com/static/images/become_sponsor.svg)](https://docs.rsshub.app/support/)
@@ -43,12 +45,12 @@ Logo designer [sheldonrrr](https://dribbble.com/sheldonrrr)
 
 ### Backers
 
-<a href="https://www.cloudflare.com" target="_blank"><img height="50px" src="https://i.imgur.com/7Ph27Fq.png"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="https://www.netlify.com" target="_blank"><img height="40px" src="https://i.imgur.com/cU01915.png"></a>
+<a href="https://www.cloudflare.com" target="_blank"><img height="50px" src="https://i.imgur.com/7Ph27Fq.png"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="https://www.netlify.com" target="_blank"><img height="40px" src="https://i.imgur.com/cU01915.png"></a>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<a href="https://1password.com" target="_blank"><img height="40px" src="https://i.imgur.com/a2XjflO.png"></a>
 
 ## Related Projects
 
 -   [RSSHub Radar](https://github.com/DIYgod/RSSHub-Radar) | A browser extension that can help you quickly discover and subscribe to the RSS and RSSHub of current websites.
--   [RSSBud](https://github.com/Cay-Zhang/RSSBud) ([TestFlight](https://testflight.apple.com/join/rjCVzzHP)) | RSSHub Radar for iOS platform, designed specifically for mobile ecosystem optimization.
+-   [RSSBud](https://github.com/Cay-Zhang/RSSBud) | RSSHub Radar for iOS platform, designed specifically for mobile ecosystem optimization.
 -   [RSSAid](https://github.com/LeetaoGoooo/RSSAid) | RSSHub Radar for Android platform built with Flutter.
 -   [DocSearch](https://github.com/Fatpandac/DocSearch) | Link RSSHub DocSearch into Raycast
 
@@ -78,7 +80,7 @@ Recurring donors will be rewarded via express issue response, or even have your
 -   Become a Sponser on [Patreon](https://www.patreon.com/DIYgod)
 -   Become a Sponser on [Open Collective](https://opencollective.com/RSSHub)
 -   Become a Sponser on [爱发电](https://afdian.net/@diygod)
--   Contact us directly: i@diygod.me
+-   Contact us directly: <i@diygod.me>
 
 ### One-time Donation
 

package/lib/config.js

@@ -1,12 +1,12 @@
 require('dotenv').config();
 const randUserAgent = require('./utils/rand-user-agent');
+const got = require('got');
 let envs = process.env;
 let value;
 const TRUE_UA = 'RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)';
 
 const calculateValue = () => {
     const bilibili_cookies = {};
-    const twitter_tokens = {};
     const email_config = {};
     const discuz_cookies = {};
     const medium_cookies = {};
@@ -16,9 +16,6 @@ const calculateValue = () => {
         if (name.startsWith('BILIBILI_COOKIE_')) {
             const uid = name.slice(16);
             bilibili_cookies[uid] = envs[name];
-        } else if (name.startsWith('TWITTER_TOKEN_')) {
-            const id = name.slice(14);
-            twitter_tokens[id] = envs[name];
         } else if (name.startsWith('EMAIL_CONFIG_')) {
             const id = name.slice(13);
             email_config[id] = envs[name];
@@ -222,6 +219,9 @@ const calculateValue = () => {
         lastfm: {
             api_key: envs.LASTFM_API_KEY,
         },
+        lightnovel: {
+            cookie: envs.SECURITY_KEY,
+        },
         manhuagui: {
             cookie: envs.MHGUI_COOKIE,
         },
@@ -295,10 +295,8 @@ const calculateValue = () => {
             cookie: envs.TOPHUB_COOKIE,
         },
         twitter: {
-            consumer_key: envs.TWITTER_CONSUMER_KEY,
-            consumer_secret: envs.TWITTER_CONSUMER_SECRET,
-            tokens: twitter_tokens,
-            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION && envs.TWITTER_WEBAPI_AUTHORIZAION.split(','),
+            oauthTokens: envs.TWITTER_OAUTH_TOKEN?.split(','),
+            oauthTokenSecrets: envs.TWITTER_OAUTH_TOKEN_SECRET?.split(','),
         },
         weibo: {
             app_key: envs.WEIBO_APP_KEY,
@@ -335,6 +333,21 @@ const calculateValue = () => {
 };
 calculateValue();
 
+if (envs.REMOTE_CONFIG) {
+    got.get(envs.REMOTE_CONFIG)
+        .then((response) => {
+            const data = JSON.parse(response.body);
+            if (data) {
+                envs = Object.assign(envs, data);
+                calculateValue();
+                require('@/utils/logger').info('Remote config loaded.');
+            }
+        })
+        .catch((error) => {
+            require('@/utils/logger').error('Remote config load failed.', error);
+        });
+}
+
 module.exports = {
     set: (env) => {
         envs = Object.assign(process.env, env);

package/lib/middleware/cache/index.js

@@ -1,4 +1,4 @@
-const md5 = require('@/utils/md5');
+const xxhash = require('xxhash-wasm');
 const config = require('@/config').value;
 const logger = require('@/utils/logger');
 const { RequestInProgressError } = require('@/errors');
@@ -81,8 +81,9 @@ module.exports = function (app) {
     };
 
     return async (ctx, next) => {
-        const key = 'koa-redis-cache:' + md5(ctx.request.path);
-        const controlKey = 'path-requested:' + md5(ctx.request.path);
+        const { h64ToString } = await xxhash();
+        const key = 'rsshub:koa-redis-cache:' + h64ToString(ctx.request.path);
+        const controlKey = 'rsshub:path-requested:' + h64ToString(ctx.request.path);
 
         if (!status.available) {
             return next();

package/lib/middleware/cache/redis.js

@@ -19,10 +19,10 @@ redisClient.on('connect', () => {
 });
 
 const getCacheTtlKey = (key) => {
-    if (key.startsWith('cacheTtl:')) {
-        throw Error('"cacheTtl:" prefix is reserved for the internal usage, please change your cache key'); // blocking any attempt to get/set the cacheTtl
+    if (key.startsWith('rsshub:cacheTtl:')) {
+        throw Error('"rsshub:cacheTtl:" prefix is reserved for the internal usage, please change your cache key'); // blocking any attempt to get/set the cacheTtl
     }
-    return `cacheTtl:${key}`;
+    return `rsshub:cacheTtl:${key}`;
 };
 
 module.exports = {

package/lib/middleware/onerror.js

@@ -39,6 +39,14 @@ module.exports = async (ctx, next) => {
             });
         }
     } catch (err) {
+        if (err instanceof Error && !err.stack.split('\n')[1].includes('lib/middleware/parameter.js')) {
+            // Append v2 route path if a route throws an error
+            // since koa-mount will remove the mount path from ctx.request.path
+            // https://github.com/koajs/mount/issues/62
+            ctx.request.path = (ctx.mountPath ?? '') + ctx.request.path;
+            ctx._matchedRoute = ctx._matchedRoute ? (ctx.mountPath ?? '') + ctx._matchedRoute : ctx._matchedRoute;
+        }
+
         let message = err;
         if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) {
             message = `${err.message}: target website might be blocking our access, you can <a href="https://docs.rsshub.app/install/">host your own RSSHub instance</a> for a better usability.`;

package/lib/middleware/template.js

@@ -1,7 +1,7 @@
-const { art, json } = require('@/utils/render');
+const { art, json, rss3Ums } = require('@/utils/render');
 const path = require('path');
 const config = require('@/config').value;
-const typeRegex = /\.(atom|rss|debug\.json|json|\d+\.debug\.html)$/;
+const typeRegex = /\.(atom|rss|ums|debug\.json|json|\d+\.debug\.html)$/;
 const { collapseWhitespace, convertDateToISO8601 } = require('@/utils/common-utils');
 
 module.exports = async (ctx, next) => {
@@ -16,111 +16,105 @@ module.exports = async (ctx, next) => {
 
     const outputType = ctx.state.type[1] || 'rss';
 
-    if (outputType === 'debug.json' && config.debugInfo) {
-        ctx.set({
-            'Content-Type': 'application/json; charset=UTF-8',
-        });
-        if (ctx.state.json) {
-            ctx.body = JSON.stringify(ctx.state.json, null, 4);
-        } else {
-            ctx.body = JSON.stringify({ message: 'plugin does not set debug json' });
-        }
-    }
-    if (outputType.endsWith('.debug.html') && config.debugInfo) {
-        ctx.set({
-            'Content-Type': 'text/html; charset=UTF-8',
-        });
-
-        const index = parseInt(outputType.match(/(\d+)\.debug\.html$/)[1]);
-        if (!(ctx.state.data && ctx.state.data.item && ctx.state.data.item[index])) {
-            ctx.body = `ctx.state.data.item[${index}] not found`;
-        } else {
-            ctx.body = ctx.state.data.item[index].description;
+    // only enable when debugInfo=true
+    if (config.debugInfo) {
+        if (outputType === 'debug.json') {
+            ctx.set({
+                'Content-Type': 'application/json; charset=UTF-8',
+            });
+            if (ctx.state.json) {
+                ctx.body = JSON.stringify(ctx.state.json, null, 4);
+            } else {
+                ctx.body = JSON.stringify({ message: 'plugin does not set debug json' });
+            }
         }
-    }
 
-    if (outputType === 'json') {
-        ctx.set({ 'Content-Type': 'application/feed+json; charset=UTF-8' });
+        if (outputType.endsWith('.debug.html')) {
+            ctx.set({
+                'Content-Type': 'text/html; charset=UTF-8',
+            });
+
+            const index = parseInt(outputType.match(/(\d+)\.debug\.html$/)[1]);
+            if (!(ctx.state.data && ctx.state.data.item && ctx.state.data.item[index])) {
+                ctx.body = `ctx.state.data.item[${index}] not found`;
+            } else {
+                ctx.body = ctx.state.data.item[index].description;
+            }
+        }
     }
 
     if (!ctx.body) {
-        let template;
-
-        switch (outputType) {
-            case 'atom':
-                template = path.resolve(__dirname, '../views/atom.art');
-                break;
-            case 'rss':
-                template = path.resolve(__dirname, '../views/rss.art');
-                break;
-            default:
-                template = path.resolve(__dirname, '../views/rss.art');
-                break;
-        }
+        const templateName = outputType === 'atom' ? 'atom.art' : 'rss.art';
+        const template = path.resolve(__dirname, `../views/${templateName}`);
 
         if (ctx.state.data) {
-            ctx.state.data.title = collapseWhitespace(ctx.state.data.title);
-            ctx.state.data.subtitle = collapseWhitespace(ctx.state.data.subtitle);
-            ctx.state.data.author = collapseWhitespace(ctx.state.data.author);
-
-            ctx.state.data.item &&
-                ctx.state.data.item.forEach((item) => {
-                    if (item.title) {
-                        item.title = collapseWhitespace(item.title);
-                        // trim title length
-                        for (let length = 0, i = 0; i < item.title.length; i++) {
-                            length += Buffer.from(item.title[i]).length !== 1 ? 2 : 1;
-                            if (length > config.titleLengthLimit) {
-                                item.title = `${item.title.slice(0, i)}...`;
-                                break;
-                            }
-                        }
+            const collapseWhitespaceForProperties = (properties, obj) => {
+                properties.forEach((prop) => {
+                    if (obj[prop]) {
+                        obj[prop] = collapseWhitespace(obj[prop]);
                     }
-
-                    if (typeof item.author === 'string') {
-                        item.author = collapseWhitespace(item.author);
-                    } else if (typeof item.author === 'object' && item.author !== null) {
-                        for (const a of item.author) {
-                            a.name = collapseWhitespace(a.name);
-                        }
-                        if (outputType !== 'json') {
-                            item.author = item.author.map((a) => a.name).join(', ');
+                });
+            };
+
+            collapseWhitespaceForProperties(['title', 'subtitle', 'author'], ctx.state.data);
+
+            ctx.state.data.item?.forEach((item) => {
+                if (item.title) {
+                    item.title = collapseWhitespace(item.title);
+                    // trim title length
+                    for (let length = 0, i = 0; i < item.title.length; i++) {
+                        length += Buffer.from(item.title[i]).length !== 1 ? 2 : 1;
+                        if (length > config.titleLengthLimit) {
+                            item.title = `${item.title.slice(0, i)}...`;
+                            break;
                         }
                     }
-
-                    if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && item.itunes_duration.indexOf(':') === -1) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) {
-                        item.itunes_duration = +item.itunes_duration;
-                        item.itunes_duration =
-                            Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2);
-                    }
-
-                    if (outputType !== 'rss') {
-                        item.pubDate = convertDateToISO8601(item.pubDate);
-                        item.updated = convertDateToISO8601(item.updated);
+                }
+
+                if (typeof item.author === 'string') {
+                    item.author = collapseWhitespace(item.author);
+                } else if (typeof item.author === 'object' && item.author !== null) {
+                    item.author.forEach((a) => (a.name = collapseWhitespace(a.name)));
+                    if (outputType !== 'json') {
+                        item.author = item.author.map((a) => a.name).join(', ');
                     }
-                });
+                }
+
+                if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && item.itunes_duration.indexOf(':') === -1) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) {
+                    item.itunes_duration = +item.itunes_duration;
+                    item.itunes_duration =
+                        Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2);
+                }
+
+                if (outputType !== 'rss') {
+                    item.pubDate = convertDateToISO8601(item.pubDate);
+                    item.updated = convertDateToISO8601(item.updated);
+                }
+            });
         }
 
-        const routeTtl = (config.cache.routeExpire / 60) | 0;
-
+        const currentDate = new Date();
         const data = {
-            lastBuildDate: new Date().toUTCString(),
-            updated: new Date().toISOString(),
-            ttl: routeTtl,
+            lastBuildDate: currentDate.toUTCString(),
+            updated: currentDate.toISOString(),
+            ttl: (config.cache.routeExpire / 60) | 0,
             atomlink: ctx.request.href,
             ...ctx.state.data,
         };
+
         if (config.isPackage) {
             ctx.body = data;
             return;
         }
-        if (!template) {
-            return;
-        }
-        if (outputType !== 'json') {
+
+        if (outputType === 'ums') {
+            ctx.set({ 'Content-Type': 'application/json; charset=UTF-8' });
+            ctx.body = rss3Ums(data);
+        } else if (outputType === 'json') {
+            ctx.set({ 'Content-Type': 'application/feed+json; charset=UTF-8' });
+            ctx.body = json(data);
+        } else {
             ctx.body = art(template, data);
-            return;
         }
-        ctx.body = json(data);
     }
 };

package/lib/radar-rules.js

@@ -288,18 +288,6 @@ module.exports = {
         ],
     },
 
-    'chinatimes.com': {
-        _name: '中時電子報',
-        www: [
-            {
-                title: '新聞',
-                docs: 'https://docs.rsshub.app/routes/traditional-media#zhong-shi-dian-zi-bao',
-                source: '/:caty',
-                target: (params) => '/chinatimes/' + params.caty,
-            },
-        ],
-    },
-
     'govopendata.com': {
         _name: '新闻联播文字版',
         cn: [
@@ -323,17 +311,6 @@ module.exports = {
             },
         ],
     },
-    'xiaomi.cn': {
-        _name: '小米社区',
-        www: [
-            {
-                title: '圈子',
-                docs: 'https://docs.rsshub.app/routes/bbs#xiao-mi-she-qu',
-                source: '/board/:boardId',
-                target: '/mi/bbs/board/:boardId',
-            },
-        ],
-    },
     'suzhou.gov.cn': {
         _name: '苏州市政府',
         www: [
@@ -857,21 +834,6 @@ module.exports = {
             },
         ],
     },
-    'popiapp.cn': {
-        _name: 'Popi 提问箱',
-        www: [
-            {
-                title: '提问箱新回答',
-                docs: 'https://docs.rsshub.app/routes/social-media#popi-ti-wen-xiang',
-                source: '/:id',
-                target: (params) => {
-                    if (params.id) {
-                        return '/popiask/:id';
-                    }
-                },
-            },
-        ],
-    },
     'nppa.gov.cn': {
         _name: '国家新闻出版署',
         www: [
@@ -940,17 +902,6 @@ module.exports = {
             },
         ],
     },
-    'buaq.net': {
-        _name: '不安全资讯',
-        '.': [
-            {
-                title: '不安全资讯',
-                docs: 'http://docs.rsshub.app/routes/new-media#bu-an-quan',
-                source: '/',
-                target: '/buaq',
-            },
-        ],
-    },
     'jian-ning.com': {
         _name: '建宁闲谈',
         '.': [
@@ -962,50 +913,6 @@ module.exports = {
             },
         ],
     },
-    'matataki.io': {
-        _name: 'matataki',
-        www: [
-            {
-                title: '最热作品',
-                docs: 'https://docs.rsshub.app/routes/new-media#matataki',
-                source: '/article/',
-                target: '/matataki/posts/hot',
-            },
-            {
-                title: '最新作品',
-                docs: 'https://docs.rsshub.app/routes/new-media#matataki',
-                source: '/article/latest',
-                target: '/matataki/posts/latest',
-            },
-            {
-                title: '作者创作',
-                docs: 'https://docs.rsshub.app/routes/new-media#matataki',
-                source: '/user/:uid',
-                target: (params) => `/matataki/users/${params.uid}/posts`,
-            },
-            {
-                title: 'Fan票关联作品',
-                docs: 'https://docs.rsshub.app/routes/new-media#matataki',
-                source: ['/token/:tokenId', '/token/:tokenId/circle'],
-                target: (params) => `/matataki/tokens/${params.tokenId}/posts`,
-            },
-            {
-                title: '标签关联作品',
-                docs: 'https://docs.rsshub.app/routes/new-media#matataki',
-                source: ['/tag/:tagId'],
-                target: (params, url) => {
-                    const tagName = new URL(url).searchParams.get('name');
-                    return `/matataki/tags/${params.tagId}/${tagName}/posts`;
-                },
-            },
-            {
-                title: '收藏夹',
-                docs: 'https://docs.rsshub.app/routes/new-media#matataki',
-                source: '/user/:uid/favlist/:fid',
-                target: (params) => `/matataki/users/${params.uid}/favorites/${params.fid}/posts`,
-            },
-        ],
-    },
     'huya.com': {
         _name: '虎牙直播',
         '.': [