rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/README.md

@@ -6,10 +6,11 @@
 > 🍰 Everything is RSSible
 
 [![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?logo=telegram&style=flat-square)](https://t.me/rsshub)
-[![npm publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/publish/master?label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
-[![docker publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/%5Bdocker%5D%20CI%20for%20releases/master?label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
-[![test](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/test/master?label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
+[![npm publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/npm-publish.yml?branch=master&label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
+[![docker publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/docker-release.yml?branch=master&label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
+[![test](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/test.yml?branch=master&label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
 [![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square&logo=codecov)](https://app.codecov.io/gh/DIYgod/RSSHub/branch/master)
+[![Foresight Docs](https://api-public.service.runforesight.com/api/v1/badge/success?repoId=5da20e56-cecf-4b05-9bcd-cab097fb2dca)](https://docs.runforesight.com/)
 
 ## Introduction
 
@@ -33,13 +34,21 @@ RSSHub 是一个开源、简单易用、易于扩展的 RSS 生成器，可以
 
 ### Special Sponsors
 
-<div>
-<a href="https://rss3.io/" target="_blank"><img height="60px" src="https://i.imgur.com/v7TF0jp.jpg"></a>
-</div>
+<p>
+<a href="https://rss3.io/" target="_blank"><img height="60px" src="https://i.imgur.com/vHtI4bz.png"></a>
+</p>
+
+<table width="100%">
+<tr>
+<td><img width="1000" height="0"><a href="https://www.runforesight.com/?utm_source=RSSHub&utm_medium=sponsorship"><img src="https://www.thundra.io/hubfs/RunForesight/GitHub%20Sponsorship%20banners/github-actions-observability.png"
+alt="Foresight" width="260" align="right"></a><h3>Foresight: Increase CI/CD Health & Test Performance</h3><p>
+Foresight provides full visibility and deep insights into the health and performance of your tests and CI pipelines. Assess the risk of code changes, deal with flaky tests, see workflow and test trends over time, and improve the contributing experience with Foresight.</p>
 
-### Sponsors
+[Sign up now!](https://runforesight.com/?utm_source=RSSHub&utm_medium=sponsorship)
 
-[Sayori Studio](https://t.me/SayoriStudio) . [Sion Kazama](https://blog.sion.moe) . [琚致远](https://wineso.me/) . [Rolly RSS 阅读器](https://www.coolapk.com/apk/239500) . [mokeyjay](https://www.mokeyjay.com/) . [萌开源联盟](https://www.moeunion.com) . [hooke007](https://github.com/hooke007/MPV_lazy) . [feeds.pub](https://feeds.pub) . [KINGX@安全引擎](http://cve.today/)
+</td>
+</tr>
+</table>
 
 [![](https://opencollective.com/static/images/become_sponsor.svg)](https://docs.rsshub.app/en/support/)
 

package/lib/config.js

@@ -2,6 +2,8 @@ require('dotenv').config();
 const randUserAgent = require('./utils/rand-user-agent');
 let envs = process.env;
 let value;
+const TRUE_UA = 'RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)';
+
 const calculateValue = () => {
     const bilibili_cookies = {};
     const twitter_tokens = {};
@@ -40,7 +42,8 @@ const calculateValue = () => {
         listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1
         requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数
         requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request
-        ua: envs.UA || randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }),
+        ua: envs.UA ? envs.UA : envs.NO_RANDOM_UA === 'true' || envs.NO_RANDOM_UA === '1' ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }),
+        trueUA: TRUE_UA,
         // cors request
         allowOrigin: envs.ALLOW_ORIGIN,
         // cache
@@ -134,6 +137,7 @@ const calculateValue = () => {
             ipb_pass_hash: envs.EH_IPB_PASS_HASH,
             sk: envs.EH_SK,
             igneous: envs.EH_IGNEOUS,
+            star: envs.EH_STAR,
             img_proxy: envs.EH_IMG_PROXY,
         },
         email: {
@@ -148,6 +152,9 @@ const calculateValue = () => {
             username: envs.FANFOU_USERNAME,
             password: envs.FANFOU_PASSWORD,
         },
+        fantia: {
+            cookies: envs.FANTIA_COOKIE,
+        },
         game4399: {
             cookie: envs.GAME_4399,
         },
@@ -184,6 +191,9 @@ const calculateValue = () => {
         lastfm: {
             api_key: envs.LASTFM_API_KEY,
         },
+        manhuagui: {
+            cookie: envs.MHGUI_COOKIE,
+        },
         mastodon: {
             apiHost: envs.MASTODON_API_HOST,
             accessToken: envs.MASTODON_API_ACCESS_TOKEN,
@@ -215,11 +225,14 @@ const calculateValue = () => {
             bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
             bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
             bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
-            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
+            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.re',
         },
         pkubbs: {
             cookie: envs.PKUBBS_COOKIE,
         },
+        saraba1st: {
+            cookie: envs.SARABA1ST_COOKIE,
+        },
         scboy: {
             token: envs.SCBOY_BBS_TOKEN,
         },
@@ -238,8 +251,8 @@ const calculateValue = () => {
             consumer_key: envs.TWITTER_CONSUMER_KEY,
             consumer_secret: envs.TWITTER_CONSUMER_SECRET,
             tokens: twitter_tokens,
-            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA',
-            // 此处硬编码参考自https://github.com/mikf/gallery-dl/blob/master/gallery_dl/extractor/twitter.py#L550
+            authorization: envs.TWITTER_WEBAPI_AUTHORIZAION || 'Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw',
+            // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25
         },
         weibo: {
             app_key: envs.WEIBO_APP_KEY,
@@ -271,6 +284,9 @@ const calculateValue = () => {
         zhihu: {
             cookies: envs.ZHIHU_COOKIES,
         },
+        zodgame: {
+            cookie: envs.ZODGAME_COOKIE,
+        },
     };
 };
 calculateValue();

package/lib/middleware/parameter.js

@@ -54,7 +54,9 @@ module.exports = async (ctx, next) => {
         ctx.state.data.description && (ctx.state.data.description = entities.decodeXML(ctx.state.data.description + ''));
 
         // sort items
-        ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
+        if (ctx.query.sorted !== 'false') {
+            ctx.state.data.item = ctx.state.data.item.sort((a, b) => +new Date(b.pubDate || 0) - +new Date(a.pubDate || 0));
+        }
 
         const handleItem = (item) => {
             item.title && (item.title = entities.decodeXML(item.title + ''));
@@ -179,7 +181,7 @@ module.exports = async (ctx, next) => {
                     const title = item.title || '';
                     const description = item.description || title;
                     const author = item.author || '';
-                    const category = item.category || [];
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
                     const isFilter =
                         title.match(makeRegex(ctx.query.filter)) || description.match(makeRegex(ctx.query.filter)) || author.match(makeRegex(ctx.query.filter)) || category.some((c) => c.match(makeRegex(ctx.query.filter)));
                     return isFilter;
@@ -192,7 +194,7 @@ module.exports = async (ctx, next) => {
                     const title = item.title || '';
                     const description = item.description || title;
                     const author = item.author || '';
-                    const category = item.category || [];
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
                     let isFilter = true;
                     ctx.query.filter_title && (isFilter = title.match(makeRegex(ctx.query.filter_title)));
                     ctx.query.filter_description && (isFilter = isFilter && description.match(makeRegex(ctx.query.filter_description)));
@@ -211,7 +213,7 @@ module.exports = async (ctx, next) => {
                     const title = item.title;
                     const description = item.description || title;
                     const author = item.author || '';
-                    const category = item.category || [];
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
                     let isFilter = true;
                     ctx.query.filterout_title && (isFilter = !title.match(makeRegex(ctx.query.filterout_title)));
                     ctx.query.filterout_description && (isFilter = isFilter && !description.match(makeRegex(ctx.query.filterout_description)));
@@ -250,12 +252,10 @@ module.exports = async (ctx, next) => {
                     const parsed_result = await ctx.cache.tryGet(`mercury-cache-${link}`, async () => {
                         // if parser failed, return default description and not report error
                         try {
-                            mercury_parser = mercury_parser || require('@postlight/mercury-parser');
+                            mercury_parser = mercury_parser || require('@postlight/parser');
 
-                            const res = await got(link);
-                            const $ = cheerio.load(res.data, {
-                                xmlMode: true,
-                            });
+                            const { data: res } = await got(link);
+                            const $ = cheerio.load(res);
                             const result = await mercury_parser.parse(link, {
                                 html: $.html(),
                             });
@@ -266,7 +266,7 @@ module.exports = async (ctx, next) => {
                     });
 
                     item.author = author || (parsed_result ? parsed_result.author : '');
-                    item.description = parsed_result ? parsed_result.content : description;
+                    item.description = parsed_result ? entities.decodeXML(parsed_result.content) : description;
                 });
                 await Promise.all(tasks);
             }

package/lib/middleware/template.js

@@ -1,7 +1,7 @@
-const art = require('art-template');
+const { art, json } = require('@/utils/render');
 const path = require('path');
 const config = require('@/config').value;
-const typeRegex = /\.(atom|rss|debug\.json)$/;
+const typeRegex = /\.(atom|rss|debug\.json|json)$/;
 const { collapseWhitespace, convertDateToISO8601 } = require('@/utils/common-utils');
 
 module.exports = async (ctx, next) => {
@@ -14,21 +14,25 @@ module.exports = async (ctx, next) => {
 
     await next();
 
-    if (ctx.state.type[1] === 'debug.json' && config.debugInfo) {
+    const outputType = ctx.state.type[1] || 'rss';
+
+    if (outputType === 'debug.json' && config.debugInfo) {
         ctx.set({
             'Content-Type': 'application/json; charset=UTF-8',
         });
         if (ctx.state.json) {
             ctx.body = JSON.stringify(ctx.state.json, null, 4);
         } else {
-            ctx.body = JSON.stringify({ message: 'plugin does not set json' });
+            ctx.body = JSON.stringify({ message: 'plugin does not set debug json' });
         }
     }
+    if (outputType === 'json') {
+        ctx.set({ 'Content-Type': 'application/feed+json; charset=UTF-8' });
+    }
 
     if (!ctx.body) {
         let template;
 
-        const outputType = ctx.state.type[1];
         switch (outputType) {
             case 'atom':
                 template = path.resolve(__dirname, '../views/atom.art');
@@ -60,7 +64,16 @@ module.exports = async (ctx, next) => {
                         }
                     }
 
-                    item.author = collapseWhitespace(item.author);
+                    if (typeof item.author === 'string') {
+                        item.author = collapseWhitespace(item.author);
+                    } else if (typeof item.author === 'object') {
+                        for (const a of item.author) {
+                            a.name = collapseWhitespace(a.name);
+                        }
+                        if (outputType !== 'json') {
+                            item.author = item.author.map((a) => a.name).join(', ');
+                        }
+                    }
 
                     if (item.itunes_duration && ((typeof item.itunes_duration === 'string' && item.itunes_duration.indexOf(':') === -1) || (typeof item.itunes_duration === 'number' && !isNaN(item.itunes_duration)))) {
                         item.itunes_duration = +item.itunes_duration;
@@ -68,7 +81,7 @@ module.exports = async (ctx, next) => {
                             Math.floor(item.itunes_duration / 3600) + ':' + (Math.floor((item.itunes_duration % 3600) / 60) / 100).toFixed(2).slice(-2) + ':' + (((item.itunes_duration % 3600) % 60) / 100).toFixed(2).slice(-2);
                     }
 
-                    if (outputType === 'atom') {
+                    if (outputType !== 'rss') {
                         item.pubDate = convertDateToISO8601(item.pubDate);
                         item.updated = convertDateToISO8601(item.updated);
                     }
@@ -86,10 +99,15 @@ module.exports = async (ctx, next) => {
         };
         if (config.isPackage) {
             ctx.body = data;
-        } else {
-            if (template) {
-                ctx.body = art(template, data);
-            }
+            return;
+        }
+        if (!template) {
+            return;
+        }
+        if (outputType !== 'json') {
+            ctx.body = art(template, data);
+            return;
         }
+        ctx.body = json(data);
     }
 };