rsshub 1.0.0-master.f60609e → 1.0.0-master.f607442

package/README.md

@@ -6,10 +6,12 @@
 > 🍰 Everything is RSSible
 
 [![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?logo=telegram&style=flat-square)](https://t.me/rsshub)
-[![npm publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/publish/master?label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
-[![docker publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/%5Bdocker%5D%20CI%20for%20releases/master?label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
-[![test](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/test/master?label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
+[![npm publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/npm-publish.yml?branch=master&label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
+[![docker publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/docker-release.yml?branch=master&label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
+[![test](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/test.yml?branch=master&label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
 [![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square&logo=codecov)](https://app.codecov.io/gh/DIYgod/RSSHub/branch/master)
+[![CodeFactor](https://www.codefactor.io/repository/github/diygod/rsshub/badge)](https://www.codefactor.io/repository/github/diygod/rsshub)
+[![DeepScan grade](https://deepscan.io/api/teams/6244/projects/8135/branches/92448/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=6244&pid=8135&bid=92448)
 
 ## Introduction
 
@@ -33,13 +35,9 @@ RSSHub 是一个开源、简单易用、易于扩展的 RSS 生成器，可以
 
 ### Special Sponsors
 
-<div>
-<a href="https://rss3.io/" target="_blank"><img height="60px" src="https://i.imgur.com/v7TF0jp.jpg"></a>
-</div>
-
-### Sponsors
-
-[Sayori Studio](https://t.me/SayoriStudio) . [Sion Kazama](https://blog.sion.moe) . [琚致远](https://wineso.me/) . [Rolly RSS 阅读器](https://www.coolapk.com/apk/239500) . [mokeyjay](https://www.mokeyjay.com/) . [萌开源联盟](https://www.moeunion.com) . [hooke007](https://github.com/hooke007/MPV_lazy) . [feeds.pub](https://feeds.pub) . [KINGX@安全引擎](http://cve.today/)
+<p>
+<a href="https://rss3.io/" target="_blank"><img height="50px" src="https://i.imgur.com/vHtI4bz.png"></a>
+</p>
 
 [![](https://opencollective.com/static/images/become_sponsor.svg)](https://docs.rsshub.app/en/support/)
 
@@ -58,6 +56,7 @@ Logo designer [sheldonrrr](https://dribbble.com/sheldonrrr)
 -   [RSSHub Radar](https://github.com/DIYgod/RSSHub-Radar) | 一个可以帮助你快速发现和订阅当前网站 RSS 和 RSSHub 的浏览器扩展
 -   [RSSBud](https://github.com/Cay-Zhang/RSSBud) ([TestFlight 公测](https://testflight.apple.com/join/rjCVzzHP)) | iOS 平台的 RSSHub Radar，专为移动生态优化
 -   [RSSAid](https://github.com/LeetaoGoooo/RSSAid)  | 基于 Flutter 构建的 Android 平台的 RSSHub Radar
+-   [DocSearch](https://github.com/Fatpandac/DocSearch) | Link RSSHub DocSearch into Raycast
 
 ## Join Us
 

package/lib/config.js

@@ -2,6 +2,8 @@ require('dotenv').config();
 const randUserAgent = require('./utils/rand-user-agent');
 let envs = process.env;
 let value;
+const TRUE_UA = 'RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)';
+
 const calculateValue = () => {
     const bilibili_cookies = {};
     const twitter_tokens = {};
@@ -40,7 +42,8 @@ const calculateValue = () => {
         listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1
         requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数
         requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request
-        ua: envs.UA || randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }),
+        ua: envs.UA ? envs.UA : envs.NO_RANDOM_UA === 'true' || envs.NO_RANDOM_UA === '1' ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }),
+        trueUA: TRUE_UA,
         // cors request
         allowOrigin: envs.ALLOW_ORIGIN,
         // cache
@@ -94,6 +97,7 @@ const calculateValue = () => {
         feature: {
             allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
             filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
+            allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true',
         },
         suffix: envs.SUFFIX,
         titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
@@ -120,6 +124,9 @@ const calculateValue = () => {
             username: envs.DIDA365_USERNAME,
             password: envs.DIDA365_PASSWORD,
         },
+        discord: {
+            authorization: envs.DISCORD_AUTHORIZATION,
+        },
         discuz: {
             cookies: discuz_cookies,
         },
@@ -134,6 +141,7 @@ const calculateValue = () => {
             ipb_pass_hash: envs.EH_IPB_PASS_HASH,
             sk: envs.EH_SK,
             igneous: envs.EH_IGNEOUS,
+            star: envs.EH_STAR,
             img_proxy: envs.EH_IMG_PROXY,
         },
         email: {
@@ -148,6 +156,9 @@ const calculateValue = () => {
             username: envs.FANFOU_USERNAME,
             password: envs.FANFOU_PASSWORD,
         },
+        fantia: {
+            cookies: envs.FANTIA_COOKIE,
+        },
         game4399: {
             cookie: envs.GAME_4399,
         },
@@ -177,10 +188,17 @@ const calculateValue = () => {
             username: envs.IG_USERNAME,
             password: envs.IG_PASSWORD,
             proxy: envs.IG_PROXY,
+            cookie: envs.IG_COOKIE,
+        },
+        iwara: {
+            cookie: envs.IWARA_COOKIE,
         },
         lastfm: {
             api_key: envs.LASTFM_API_KEY,
         },
+        manhuagui: {
+            cookie: envs.MHGUI_COOKIE,
+        },
         mastodon: {
             apiHost: envs.MASTODON_API_HOST,
             accessToken: envs.MASTODON_API_ACCESS_TOKEN,
@@ -207,16 +225,22 @@ const calculateValue = () => {
         pianyuan: {
             cookie: envs.PIANYUAN_COOKIE,
         },
+        pixabay: {
+            key: envs.PIXABAY_KEY,
+        },
         pixiv: {
             refreshToken: envs.PIXIV_REFRESHTOKEN,
             bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
             bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
             bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
-            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
+            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.re',
         },
         pkubbs: {
             cookie: envs.PKUBBS_COOKIE,
         },
+        saraba1st: {
+            cookie: envs.SARABA1ST_COOKIE,
+        },
         scboy: {
             token: envs.SCBOY_BBS_TOKEN,
         },
@@ -235,8 +259,8 @@ const calculateValue = () => {
             consumer_key: envs.TWITTER_CONSUMER_KEY,
             consumer_secret: envs.TWITTER_CONSUMER_SECRET,
             tokens: twitter_tokens,
-            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
-            // 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
+            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw',
+            // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25
         },
         weibo: {
             app_key: envs.WEIBO_APP_KEY,
@@ -262,12 +286,12 @@ const calculateValue = () => {
             clientSecret: envs.YOUTUBE_CLIENT_SECRET,
             refreshToken: envs.YOUTUBE_REFRESH_TOKEN,
         },
-        yuque: {
-            token: envs.YUQUE_TOKEN,
-        },
         zhihu: {
             cookies: envs.ZHIHU_COOKIES,
         },
+        zodgame: {
+            cookie: envs.ZODGAME_COOKIE,
+        },
     };
 };
 calculateValue();

package/lib/middleware/cache/index.js

@@ -56,6 +56,9 @@ module.exports = function (app) {
     app.context.cache = {
         ...cacheModule,
         tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire, refresh = true) => {
+            if (typeof key !== 'string') {
+                throw Error('Cache key must be a string');
+            }
             let v = await get(key, refresh);
             if (!v) {
                 v = await getValueFunc();

package/lib/middleware/parameter.js

@@ -54,7 +54,9 @@ module.exports = async (ctx, next) => {
         ctx.state.data.description && (ctx.state.data.description = entities.decodeXML(ctx.state.data.description + ''));
 
         // sort items
-        ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
+        if (ctx.query.sorted !== 'false') {
+            ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
+        }
 
         const handleItem = (item) => {
             item.title && (item.title = entities.decodeXML(item.title + ''));
@@ -137,6 +139,13 @@ module.exports = async (ctx, next) => {
                 });
 
                 item.description = $('body').html() + '' + (config.suffix || '');
+
+                if (item._extra?.links && $('.rsshub-quote').length) {
+                    item._extra?.links?.map((e) => {
+                        e.content_html = $.html($('.rsshub-quote'));
+                        return e;
+                    });
+                }
             }
             return item;
         };
@@ -179,26 +188,30 @@ module.exports = async (ctx, next) => {
                     const title = item.title || '';
                     const description = item.description || title;
                     const author = item.author || '';
-                    const isFilter = title.match(makeRegex(ctx.query.filter)) || description.match(makeRegex(ctx.query.filter)) || author.match(makeRegex(ctx.query.filter));
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
+                    const isFilter =
+                        title.match(makeRegex(ctx.query.filter)) || description.match(makeRegex(ctx.query.filter)) || author.match(makeRegex(ctx.query.filter)) || category.some((c) => c.match(makeRegex(ctx.query.filter)));
                     return isFilter;
                 });
             }
 
-            // 启用filter参数时，无效filter_title/description/author
-            if (!ctx.query.filter && (ctx.query.filter_title || ctx.query.filter_description || ctx.query.filter_author)) {
+            // 启用filter参数时，无效filter_title/description/author/category
+            if (!ctx.query.filter && (ctx.query.filter_title || ctx.query.filter_description || ctx.query.filter_author || ctx.query.filter_category)) {
                 ctx.state.data.item = ctx.state.data.item.filter((item) => {
                     const title = item.title || '';
                     const description = item.description || title;
                     const author = item.author || '';
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
                     let isFilter = true;
                     ctx.query.filter_title && (isFilter = title.match(makeRegex(ctx.query.filter_title)));
                     ctx.query.filter_description && (isFilter = isFilter && description.match(makeRegex(ctx.query.filter_description)));
                     ctx.query.filter_author && (isFilter = isFilter && author.match(makeRegex(ctx.query.filter_author)));
+                    ctx.query.filter_category && (isFilter = isFilter && category.some((c) => c.match(makeRegex(ctx.query.filter_category))));
                     return isFilter;
                 });
             }
 
-            if (ctx.query.filterout || ctx.query.filterout_title || ctx.query.filterout_description || ctx.query.filterout_author) {
+            if (ctx.query.filterout || ctx.query.filterout_title || ctx.query.filterout_description || ctx.query.filterout_author || ctx.query.filterout_category) {
                 if (ctx.query.filterout) {
                     ctx.query.filterout_title = ctx.query.filterout;
                     ctx.query.filterout_description = ctx.query.filterout;
@@ -207,10 +220,12 @@ module.exports = async (ctx, next) => {
                     const title = item.title;
                     const description = item.description || title;
                     const author = item.author || '';
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
                     let isFilter = true;
                     ctx.query.filterout_title && (isFilter = !title.match(makeRegex(ctx.query.filterout_title)));
                     ctx.query.filterout_description && (isFilter = isFilter && !description.match(makeRegex(ctx.query.filterout_description)));
                     ctx.query.filterout_author && (isFilter = isFilter && !author.match(makeRegex(ctx.query.filterout_author)));
+                    ctx.query.filterout_category && (isFilter = isFilter && !category.some((c) => c.match(makeRegex(ctx.query.filterout_category))));
                     return isFilter;
                 });
             }
@@ -244,12 +259,10 @@ module.exports = async (ctx, next) => {
                     const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
                         // if parser failed, return default description and not report error
                         try {
-                            mercury_parser = mercury_parser || require('@postlight/mercury-parser');
+                            mercury_parser = mercury_parser || require('@postlight/parser');
 
-                            const res = await got(link);
-                            const $ = cheerio.load(res.data, {
-                                xmlMode: true,
-                            });
+                            const { data: res } = await got(link);
+                            const $ = cheerio.load(res);
                             const result = await mercury_parser.parse(link, {
                                 html: $.html(),
                             });
@@ -260,7 +273,7 @@ module.exports = async (ctx, next) => {
                     });
 
                     item.author = author || (parsed_result ? parsed_result.author : '');
-                    item.description = parsed_result ? parsed_result.content : description;
+                    item.description = parsed_result && parsed_result.content.length > 40 ? entities.decodeXML(parsed_result.content) : description;
                 });
                 await Promise.all(tasks);
             }
@@ -300,7 +313,7 @@ module.exports = async (ctx, next) => {
                         }
                     });
                 } else {
-                    throw Error(`Invalid parameter <code>brief=${ctx.query.brief}</code>. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
+                    throw Error(`Invalid parameter brief. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
                 }
             }
 

package/lib/middleware/template.js

@@ -1,7 +1,7 @@
-const art = require('art-template');
+const { art, json } = require('@/utils/render');
 const path = require('path');
 const config = require('@/config').value;
-const typeRegex = /\.(atom|rss|debug\.json)$/;
+const typeRegex = /\.(atom|rss|debug\.json|json)$/;
 const { collapseWhitespace, convertDateToISO8601 } = require('@/utils/common-utils');
 
 module.exports = async (ctx, next) => {
@@ -14,21 +14,25 @@ module.exports = async (ctx, next) => {
 
     await next();
 
-    if (ctx.state.type[1] === 'debug.json' && config.debugInfo) {
+    const outputType = ctx.state.type[1] || 'rss';
+
+    if (outputType === 'debug.json' && config.debugInfo) {
         ctx.set({
             'Content-Type': 'application/json; charset=UTF-8',
         });
         if (ctx.state.json) {
             ctx.body = JSON.stringify(ctx.state.json, null, 4);
         } else {
-            ctx.body = JSON.stringify({ message: 'plugin does not set json' });
+            ctx.body = JSON.stringify({ message: 'plugin does not set debug json' });
         }
     }
+    if (outputType === 'json') {
+        ctx.set({ 'Content-Type': 'application/feed+json; charset=UTF-8' });
+    }
 
     if (!ctx.body) {
         let template;
 
-        const outputType = ctx.state.type[1];
         switch (outputType) {
             case 'atom':
                 template = path.resolve(__dirname, '../views/atom.art');
@@ -60,7 +64,16 @@ module.exports = async (ctx, next) => {
                         }
                     }
 
-                    item.author = collapseWhitespace(item.author);
+                    if (typeof item.author === 'string') {
+                        item.author = collapseWhitespace(item.author);
+                    } else if (typeof item.author === 'object' && item.author !== null) {
+                        for (const a of item.author) {
+                            a.name = collapseWhitespace(a.name);
+                        }
+                        if (outputType !== 'json') {
+                            item.author = item.author.map((a) => a.name).join(', ');
+                        }
+                    }
 
                     if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && item.itunes_duration.indexOf(':') === -1) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) {
                         item.itunes_duration = +item.itunes_duration;
@@ -68,7 +81,7 @@ module.exports = async (ctx, next) => {
                             Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2);
                     }
 
-                    if (outputType === 'atom') {
+                    if (outputType !== 'rss') {
                         item.pubDate = convertDateToISO8601(item.pubDate);
                         item.updated = convertDateToISO8601(item.updated);
                     }
@@ -86,10 +99,15 @@ module.exports = async (ctx, next) => {
         };
         if (config.isPackage) {
             ctx.body = data;
-        } else {
-            if (template) {
-                ctx.body = art(template, data);
-            }
+            return;
+        }
+        if (!template) {
+            return;
+        }
+        if (outputType !== 'json') {
+            ctx.body = art(template, data);
+            return;
         }
+        ctx.body = json(data);
     }
 };