rsshub 1.0.0-master.fccb918 → 1.0.0-master.fd1d453

package/README.md

@@ -9,6 +9,7 @@
 [![npm publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/publish/master?label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
 [![docker publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/%5Bdocker%5D%20CI%20for%20releases/master?label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
 [![test](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/test/master?label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
+[![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square&logo=codecov)](https://app.codecov.io/gh/DIYgod/RSSHub/branch/master)
 
 ## Introduction
 
@@ -57,6 +58,7 @@ Logo designer [sheldonrrr](https://dribbble.com/sheldonrrr)
 -   [RSSHub Radar](https://github.com/DIYgod/RSSHub-Radar) | 一个可以帮助你快速发现和订阅当前网站 RSS 和 RSSHub 的浏览器扩展
 -   [RSSBud](https://github.com/Cay-Zhang/RSSBud) ([TestFlight 公测](https://testflight.apple.com/join/rjCVzzHP)) | iOS 平台的 RSSHub Radar，专为移动生态优化
 -   [RSSAid](https://github.com/LeetaoGoooo/RSSAid)  | 基于 Flutter 构建的 Android 平台的 RSSHub Radar
+-   [DocSearch](https://github.com/Fatpandac/DocSearch) | Link RSSHub DocSearch into Raycast
 
 ## Join Us
 

package/lib/config.js

@@ -1,8 +1,7 @@
 require('dotenv').config();
-const randUserAgent = require('@/utils/rand-user-agent');
+const randUserAgent = require('./utils/rand-user-agent');
 let envs = process.env;
 let value;
-
 const calculateValue = () => {
     const bilibili_cookies = {};
     const twitter_tokens = {};
@@ -32,6 +31,7 @@ const calculateValue = () => {
         isPackage: envs.IS_PACKAGE,
         nodeName: envs.NODE_NAME,
         puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT,
+        chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH,
         // network
         connect: {
             port: envs.PORT || 1200, // 监听端口
@@ -88,6 +88,12 @@ const calculateValue = () => {
         // feed config
         hotlink: {
             template: envs.HOTLINK_TEMPLATE,
+            includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','),
+            excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','),
+        },
+        feature: {
+            allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
+            filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
         },
         suffix: envs.SUFFIX,
         titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,
@@ -128,6 +134,8 @@ const calculateValue = () => {
             ipb_pass_hash: envs.EH_IPB_PASS_HASH,
             sk: envs.EH_SK,
             igneous: envs.EH_IGNEOUS,
+            star: envs.EH_STAR,
+            img_proxy: envs.EH_IMG_PROXY,
         },
         email: {
             config: email_config,
@@ -147,6 +155,9 @@ const calculateValue = () => {
         github: {
             access_token: envs.GITHUB_ACCESS_TOKEN,
         },
+        gitee: {
+            access_token: envs.GITEE_ACCESS_TOKEN,
+        },
         google: {
             fontsApiKey: envs.GOOGLE_FONTS_API_KEY,
         },
@@ -168,6 +179,9 @@ const calculateValue = () => {
             password: envs.IG_PASSWORD,
             proxy: envs.IG_PROXY,
         },
+        iwara: {
+            cookie: envs.IWARA_COOKIE,
+        },
         lastfm: {
             api_key: envs.LASTFM_API_KEY,
         },
@@ -194,6 +208,9 @@ const calculateValue = () => {
             username: envs.NHENTAI_USERNAME,
             password: envs.NHENTAI_PASSWORD,
         },
+        pianyuan: {
+            cookie: envs.PIANYUAN_COOKIE,
+        },
         pixiv: {
             refreshToken: envs.PIXIV_REFRESHTOKEN,
             bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
@@ -245,6 +262,9 @@ const calculateValue = () => {
         },
         youtube: {
             key: envs.YOUTUBE_KEY,
+            clientId: envs.YOUTUBE_CLIENT_ID,
+            clientSecret: envs.YOUTUBE_CLIENT_SECRET,
+            refreshToken: envs.YOUTUBE_REFRESH_TOKEN,
         },
         yuque: {
             token: envs.YUQUE_TOKEN,

package/lib/middleware/anti-hotlink.js

@@ -1,10 +1,40 @@
 const config = require('@/config').value;
 const cheerio = require('cheerio');
 const logger = require('@/utils/logger');
+const path = require('path');
+const { art } = require('@/utils/render');
 
-const interpolate = (str, obj) => str.replace(/\${([^}]+)}/g, (_, prop) => obj[prop]);
-// I don't want to keep another regex and
-// URL will be the standard way to parse URL
+const templateRegex = /\$\{([^{}]+)}/g;
+const allowedUrlProperties = ['hash', 'host', 'hostname', 'href', 'origin', 'password', 'pathname', 'port', 'protocol', 'search', 'searchParams', 'username'];
+const IframeWrapperTemplate = path.join(__dirname, 'templates/iframe.art');
+
+// match path or sub-path
+const matchPath = (path, paths) => {
+    for (const p of paths) {
+        if (path.startsWith(p) && (path.length === p.length || path[p.length] === '/')) {
+            return true;
+        }
+    }
+    return false;
+};
+
+// return ture if the path needs to be processed
+const filterPath = (path) => {
+    const include = config.hotlink.includePaths;
+    const exclude = config.hotlink.excludePaths;
+    return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude));
+};
+
+const interpolate = (str, obj) =>
+    str.replace(templateRegex, (_, prop) => {
+        let needEncode = false;
+        if (prop.endsWith('_ue')) {
+            // url encode
+            prop = prop.slice(0, -3);
+            needEncode = true;
+        }
+        return needEncode ? encodeURIComponent(obj[prop]) : obj[prop];
+    });
 const parseUrl = (str) => {
     let url;
     try {
@@ -15,40 +45,104 @@ const parseUrl = (str) => {
 
     return url;
 };
-const replaceUrls = (body, template) => {
-    const $ = cheerio.load(body, { decodeEntities: false, xmlMode: true });
-    $('img').each(function () {
-        const old_src = $(this).attr('src');
-        const url = parseUrl(old_src);
-        if (url) {
-            const new_src = interpolate(template, url);
-            $(this).attr('src', new_src);
+const replaceUrls = ($, selector, template, attribute = 'src') => {
+    $(selector).each(function () {
+        const old_src = $(this).attr(attribute);
+        if (old_src) {
+            const url = parseUrl(old_src);
+            if (url) {
+                // Cheerio will do the right thing to prohibit XSS.
+                $(this).attr(attribute, interpolate(template, url));
+            }
         }
     });
+};
+
+const wrapWithIframe = ($, selector) => {
+    $(selector).each((_, elem) => {
+        elem = $(elem);
+        elem.replaceWith(art(IframeWrapperTemplate, { content: elem.toString() }));
+    });
+};
 
-    return $.root().html();
+const process = (html, image_hotlink_template, multimedia_hotlink_template, wrap_multimedia_in_iframe) => {
+    const $ = cheerio.load(html, undefined, false);
+    if (image_hotlink_template) {
+        replaceUrls($, 'img, picture > source', image_hotlink_template);
+        replaceUrls($, 'video[poster]', image_hotlink_template, 'poster');
+    }
+    if (multimedia_hotlink_template) {
+        replaceUrls($, 'video, video > source, audio, audio > source', multimedia_hotlink_template);
+        if (!image_hotlink_template) {
+            replaceUrls($, 'video[poster]', multimedia_hotlink_template, 'poster');
+        }
+    }
+    if (wrap_multimedia_in_iframe) {
+        wrapWithIframe($, 'video, audio');
+    }
+    return $.html();
+};
+
+const validateTemplate = (template) => {
+    if (!template) {
+        return;
+    }
+    [...template.matchAll(templateRegex)].forEach((match) => {
+        const prop = match[1].endsWith('_ue') ? match[1].slice(0, -3) : match[1];
+        if (!allowedUrlProperties.includes(prop)) {
+            throw new Error(`Invalid URL property: ${prop}`);
+        }
+    });
 };
 
 module.exports = async (ctx, next) => {
     await next();
 
-    const template = config.hotlink.template;
+    let image_hotlink_template = undefined;
+    let multimedia_hotlink_template = undefined;
+    const shouldWrapInIframe = ctx.query.wrap_multimedia_in_iframe === '1';
+
+    // Read params if enabled
+    if (config.feature.allow_user_hotlink_template) {
+        // By default, the config turns these features off. Set corresponding config to
+        // true to turn this feature on.
+        // A risk is that the media URLs will be replaced by user-supplied templates,
+        // so a user could literally take the control of "where are the media from",
+        // but only in their personal-use feed URL.
+        multimedia_hotlink_template = ctx.query.multimedia_hotlink_template;
+        image_hotlink_template = ctx.query.image_hotlink_template;
+    }
+
+    // Force config hotlink template on conflict
+    if (config.hotlink.template) {
+        if (!filterPath(ctx.request.path)) {
+            image_hotlink_template = undefined;
+        } else {
+            image_hotlink_template = config.hotlink.template;
+        }
+    }
+
+    if (!image_hotlink_template && !multimedia_hotlink_template && !shouldWrapInIframe) {
+        return;
+    }
+
+    validateTemplate(image_hotlink_template);
+    validateTemplate(multimedia_hotlink_template);
+
     // Assume that only description include image link
     // and here we will only check them in description.
-    // Use Cherrio to load the description as html and filter all
+    // Use Cheerio to load the description as html and filter all
     // image link
-    if (template) {
-        if (ctx.state.data) {
-            if (ctx.state.data.description) {
-                ctx.state.data.description = replaceUrls(ctx.state.data.description, template);
-            }
-
-            ctx.state.data.item &&
-                ctx.state.data.item.forEach((item) => {
-                    if (item.description) {
-                        item.description = replaceUrls(item.description, template);
-                    }
-                });
+    if (ctx.state.data) {
+        if (ctx.state.data.description) {
+            ctx.state.data.description = process(ctx.state.data.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
         }
+
+        ctx.state.data.item &&
+            ctx.state.data.item.forEach((item) => {
+                if (item.description) {
+                    item.description = process(item.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
+                }
+            });
     }
 };

package/lib/middleware/cache/index.js

@@ -56,6 +56,9 @@ module.exports = function (app) {
     app.context.cache = {
         ...cacheModule,
         tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire, refresh = true) => {
+            if (typeof key !== 'string') {
+                throw Error('Cache key must be a string');
+            }
             let v = await get(key, refresh);
             if (!v) {
                 v = await getValueFunc();

package/lib/middleware/cache/redis.js

@@ -32,8 +32,10 @@ module.exports = {
             let [value, cacheTtl] = await redisClient.mget(key, cacheTtlKey);
             if (value && refresh) {
                 if (!cacheTtl) {
+                    // if cacheTtl is not set, that means the cache expire time is contentExpire
                     cacheTtl = config.cache.contentExpire;
-                    redisClient.set(cacheTtlKey, cacheTtl, 'EX', cacheTtl);
+                    // dont save cacheTtl to Redis, as it is the default value
+                    // redisClient.set(cacheTtlKey, cacheTtl, 'EX', cacheTtl);
                 } else {
                     redisClient.expire(cacheTtlKey, cacheTtl);
                 }
@@ -54,7 +56,10 @@ module.exports = {
             value = JSON.stringify(value);
         }
         if (key) {
-            redisClient.set(getCacheTtlKey(key), maxAge, 'EX', maxAge);
+            if (maxAge !== config.cache.contentExpire) {
+                // Only set cacheTtlKey if maxAge !== contentExpire
+                redisClient.set(getCacheTtlKey(key), maxAge, 'EX', maxAge);
+            }
             return redisClient.set(key, value, 'EX', maxAge); // setMode: https://redis.io/commands/set
         }
     },

package/lib/middleware/onerror.js

@@ -23,7 +23,7 @@ if (config.sentry.dsn) {
 try {
     gitHash = require('git-rev-sync').short();
 } catch (e) {
-    gitHash = (process.env.HEROKU_SLUG_COMMIT && process.env.HEROKU_SLUG_COMMIT.slice(0, 7)) || (process.env.VERCEL_GITHUB_COMMIT_SHA && process.env.VERCEL_GITHUB_COMMIT_SHA.slice(0, 7)) || 'unknown';
+    gitHash = (process.env.HEROKU_SLUG_COMMIT && process.env.HEROKU_SLUG_COMMIT.slice(0, 7)) || (process.env.VERCEL_GIT_COMMIT_SHA && process.env.VERCEL_GIT_COMMIT_SHA.slice(0, 7)) || 'unknown';
 }
 
 module.exports = async (ctx, next) => {
@@ -62,6 +62,7 @@ module.exports = async (ctx, next) => {
             if (err instanceof RequestInProgressError) {
                 ctx.status = 503;
                 message = err.message;
+                ctx.set('Cache-Control', `public, max-age=${config.cache.requestTimeout}`);
             } else if (ctx.status === 403) {
                 message = err.message;
             } else {

package/lib/middleware/parameter.js

@@ -3,16 +3,41 @@ const cheerio = require('cheerio');
 const { simplecc } = require('simplecc-wasm');
 const got = require('@/utils/got');
 const config = require('@/config').value;
+const RE2 = require('re2');
 
 let mercury_parser;
 
+const resolveRelativeLink = ($, elem, attr, baseUrl) => {
+    const $elem = $(elem);
+
+    if (baseUrl) {
+        try {
+            const oldAttr = $elem.attr(attr);
+            if (oldAttr) {
+                // e.g. <video><source src="https://example.com"></video> should leave <video> unchanged
+                $elem.attr(attr, new URL(oldAttr, baseUrl).href);
+            }
+        } catch (e) {
+            // no-empty
+        }
+    }
+};
+
 module.exports = async (ctx, next) => {
     await next();
 
     if (!ctx.state.data && !ctx._matchedRoute) {
-        ctx.set({
-            'Cache-Control': `public, max-age=${config.cache.routeExpire * 100}`,
-        });
+        // Given that the official demo has a cache TTL of 2h, a "wrong path" page will be cached by CloudFlare for
+        // 200h (8.33d).
+        // What makes it worse is that the documentation contains status badges to detect the availability of routes,
+        // but the documentation is updated more timely than the official demo, so the every example path of every
+        // new route will probably have a "wrong path" page cached for at least 200h soon after accepted. That is to
+        // say, the example paths of a new route will probably be unavailable on the public demo in the first 200h
+        // after accepted.
+        // As a conclusion, the next 3 lines has been commented out. (exactly the same behavior as any internal error)
+        // ctx.set({
+        //     'Cache-Control': `public, max-age=${config.cache.routeExpire * 100}`,
+        // });
         throw Error('wrong path');
     }
 
@@ -68,50 +93,49 @@ module.exports = async (ctx, next) => {
 
                 $('script').remove();
 
-                $('a').each((_, ele) => {
-                    const $ele = $(ele);
-
-                    // absolute link
-                    if (baseUrl) {
-                        try {
-                            $ele.attr('href', new URL($ele.attr('href'), baseUrl).href);
-                        } catch (e) {
-                            // no-empty
-                        }
-                    }
-                });
-
                 $('img').each((_, ele) => {
                     const $ele = $(ele);
 
                     // fix lazyload
                     if (!$ele.attr('src')) {
-                        for (const key in ele.attribs) {
-                            const value = ele.attribs[key].trim();
-                            if (['.gif', '.png', '.jpg', '.webp'].some((suffix) => value.includes(suffix))) {
-                                $ele.attr('src', value);
-                                break;
+                        const lazySrc = $ele.attr('data-src') || $ele.attr('data-original');
+                        if (lazySrc) {
+                            $ele.attr('src', lazySrc);
+                        } else {
+                            for (const key in ele.attribs) {
+                                const value = ele.attribs[key].trim();
+                                if (['.gif', '.png', '.jpg', '.webp'].some((suffix) => value.includes(suffix))) {
+                                    $ele.attr('src', value);
+                                    break;
+                                }
                             }
                         }
                     }
 
-                    // absolute link
-                    if (baseUrl) {
-                        try {
-                            $ele.attr('src', new URL($ele.attr('src'), baseUrl).href);
-                        } catch (e) {
-                            // no-empty
-                        }
-                    }
-
-                    // referrerpolicy
-                    $ele.attr('referrerpolicy', 'no-referrer');
-
                     // redundant attributes
                     ['onclick', 'onerror', 'onload'].forEach((e) => {
                         $ele.removeAttr(e);
                     });
                 });
+
+                // resolve relative link & fix referrer policy
+                // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
+                // https://www.w3schools.com/tags/att_href.asp
+                $('a, area').each((_, elem) => {
+                    resolveRelativeLink($, elem, 'href', baseUrl);
+                    // $(elem).attr('rel', 'noreferrer');  // currently no such a need
+                });
+                // https://www.w3schools.com/tags/att_src.asp
+                $('img, video, audio, source, iframe, embed, track').each((_, elem) => {
+                    resolveRelativeLink($, elem, 'src', baseUrl);
+                });
+                $('video[poster]').each((_, elem) => {
+                    resolveRelativeLink($, elem, 'poster', baseUrl);
+                });
+                $('img, iframe').each((_, elem) => {
+                    $(elem).attr('referrerpolicy', 'no-referrer');
+                });
+
                 item.description = $('body').html() + '' + (config.suffix || '');
             }
             return item;
@@ -128,10 +152,25 @@ module.exports = async (ctx, next) => {
             // filter
             const makeRegex = (string) => {
                 // default: case_senstivie = true
+                const engine = config.feature.filter_regex_engine;
                 if (ctx.query.filter_case_sensitive === 'false') {
-                    return new RegExp(string, 'i');
+                    switch (engine) {
+                        case 'regexp':
+                            return new RegExp(string, 'i');
+                        case 're2':
+                            return new RE2(string, 'i');
+                        default:
+                            throw Error(`Invalid Engine Value: ${engine}, please check your config.`);
+                    }
                 } else {
-                    return new RegExp(string);
+                    switch (engine) {
+                        case 'regexp':
+                            return new RegExp(string);
+                        case 're2':
+                            return new RE2(string);
+                        default:
+                            throw Error(`Invalid Engine Value: ${engine}, please check your config.`);
+                    }
                 }
             };
 
@@ -140,26 +179,30 @@ module.exports = async (ctx, next) => {
                     const title = item.title || '';
                     const description = item.description || title;
                     const author = item.author || '';
-                    const isFilter = title.match(makeRegex(ctx.query.filter)) || description.match(makeRegex(ctx.query.filter)) || author.match(makeRegex(ctx.query.filter));
+                    const category = item.category || [];
+                    const isFilter =
+                        title.match(makeRegex(ctx.query.filter)) || description.match(makeRegex(ctx.query.filter)) || author.match(makeRegex(ctx.query.filter)) || category.some((c) => c.match(makeRegex(ctx.query.filter)));
                     return isFilter;
                 });
             }
 
-            // 启用filter参数时，无效filter_title/description/author
-            if (!ctx.query.filter && (ctx.query.filter_title || ctx.query.filter_description || ctx.query.filter_author)) {
+            // 启用filter参数时，无效filter_title/description/author/category
+            if (!ctx.query.filter && (ctx.query.filter_title || ctx.query.filter_description || ctx.query.filter_author || ctx.query.filter_category)) {
                 ctx.state.data.item = ctx.state.data.item.filter((item) => {
                     const title = item.title || '';
                     const description = item.description || title;
                     const author = item.author || '';
+                    const category = item.category || [];
                     let isFilter = true;
                     ctx.query.filter_title && (isFilter = title.match(makeRegex(ctx.query.filter_title)));
                     ctx.query.filter_description && (isFilter = isFilter && description.match(makeRegex(ctx.query.filter_description)));
                     ctx.query.filter_author && (isFilter = isFilter && author.match(makeRegex(ctx.query.filter_author)));
+                    ctx.query.filter_category && (isFilter = isFilter && category.some((c) => c.match(makeRegex(ctx.query.filter_category))));
                     return isFilter;
                 });
             }
 
-            if (ctx.query.filterout || ctx.query.filterout_title || ctx.query.filterout_description || ctx.query.filterout_author) {
+            if (ctx.query.filterout || ctx.query.filterout_title || ctx.query.filterout_description || ctx.query.filterout_author || ctx.query.filterout_category) {
                 if (ctx.query.filterout) {
                     ctx.query.filterout_title = ctx.query.filterout;
                     ctx.query.filterout_description = ctx.query.filterout;
@@ -168,10 +211,12 @@ module.exports = async (ctx, next) => {
                     const title = item.title;
                     const description = item.description || title;
                     const author = item.author || '';
+                    const category = item.category || [];
                     let isFilter = true;
                     ctx.query.filterout_title && (isFilter = !title.match(makeRegex(ctx.query.filterout_title)));
                     ctx.query.filterout_description && (isFilter = isFilter && !description.match(makeRegex(ctx.query.filterout_description)));
                     ctx.query.filterout_author && (isFilter = isFilter && !author.match(makeRegex(ctx.query.filterout_author)));
+                    ctx.query.filterout_category && (isFilter = isFilter && !category.some((c) => c.match(makeRegex(ctx.query.filterout_category))));
                     return isFilter;
                 });
             }
@@ -264,6 +309,8 @@ module.exports = async (ctx, next) => {
                     throw Error(`Invalid parameter <code>brief=${ctx.query.brief}</code>. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
                 }
             }
+
+            // some parameters are processed in `anti-hotlink.js`
         }
     }
 };

package/lib/middleware/templates/iframe.art

@@ -0,0 +1,14 @@
+<iframe referrerpolicy="no-referrer" width=100% height=150vh frameborder=0 marginheight=0 marginwidth=0
+style="border:0; margin:0; padding:0; width:100%; height:150vh;"
+srcdoc="
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta name=&quot;referrer&quot; content=&quot;no-referrer&quot;>
+  </head>
+  <body>
+    {{ content }}
+  </body>
+</html>
+">
+</iframe>