npm - rsshub - Versions diffs - 1.0.0-master.fc5a2f6 → 1.0.0-master.fc648b5 - Mend

rsshub 1.0.0-master.fc5a2f6 → 1.0.0-master.fc648b5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (292) hide show

package/lib/config.js +4 -0
package/lib/middleware/anti-hotlink.js +90 -22
package/lib/middleware/parameter.js +65 -32
package/lib/middleware/templates/iframe.art +14 -0
package/lib/radar-rules.js +0 -92
package/lib/router.js +22 -22
package/lib/routes/agefans/update.js +1 -1
package/lib/routes/novel/biqugeinfo.js +2 -2
package/lib/routes/universities/uestc/sice.js +12 -33
package/lib/routes/weibo/user.js +10 -19
package/lib/v2/19lou/index.js +74 -0
package/lib/v2/19lou/maintainer.js +3 -0
package/lib/v2/19lou/radar.js +29 -0
package/lib/v2/19lou/router.js +3 -0
package/lib/v2/36kr/router.js +1 -1
package/lib/v2/36kr/user.js +47 -22
package/lib/v2/acfun/article.js +100 -0
package/lib/{routes → v2}/acfun/bangumi.js +7 -7
package/lib/v2/acfun/maintainer.js +5 -0
package/lib/v2/acfun/radar.js +25 -0
package/lib/v2/acfun/router.js +5 -0
package/lib/v2/acfun/video.js +45 -0
package/lib/v2/agora0/index.js +54 -0
package/lib/v2/agora0/maintainer.js +3 -0
package/lib/v2/agora0/radar.js +13 -0
package/lib/v2/agora0/router.js +3 -0
package/lib/v2/apnews/maintainer.js +3 -0
package/lib/v2/apnews/radar.js +13 -0
package/lib/v2/apnews/router.js +3 -0
package/lib/v2/apnews/templates/description.art +14 -0
package/lib/v2/apnews/topics.js +44 -0
package/lib/v2/appleinsider/index.js +59 -0
package/lib/v2/appleinsider/maintainer.js +3 -0
package/lib/v2/appleinsider/radar.js +13 -0
package/lib/v2/appleinsider/router.js +3 -0
package/lib/v2/cbnweek/index.js +46 -0
package/lib/v2/cbnweek/maintainer.js +3 -0
package/lib/v2/cbnweek/radar.js +13 -0
package/lib/v2/cbnweek/router.js +3 -0
package/lib/v2/ccac/news.js +12 -3
package/lib/v2/cde/index.js +2 -2
package/lib/v2/cde/maintainer.js +1 -0
package/lib/v2/cde/radar.js +22 -4
package/lib/v2/cde/router.js +1 -0
package/lib/v2/cde/templates/xxgk/breakthroughCure.art +18 -0
package/lib/v2/cde/templates/xxgk/cliniCal.art +16 -0
package/lib/v2/cde/templates/xxgk/priorityApproval.art +18 -0
package/lib/v2/cde/utils.js +2 -2
package/lib/v2/cde/xxgk.js +73 -0
package/lib/v2/chinadegrees/maintainer.js +3 -0
package/lib/v2/chinadegrees/province.js +66 -0
package/lib/v2/chinadegrees/radar.js +13 -0
package/lib/v2/chinadegrees/router.js +3 -0
package/lib/v2/chinadegrees/templates/description.art +4 -0
package/lib/v2/cnjxol/index.js +94 -0
package/lib/v2/cnjxol/maintainer.js +4 -0
package/lib/v2/cnjxol/radar.js +19 -0
package/lib/v2/cnjxol/router.js +3 -0
package/lib/v2/cnjxol/templates/description.art +2 -0
package/lib/v2/cscse/maintainer.js +3 -0
package/lib/v2/cscse/radar.js +13 -0
package/lib/v2/cscse/router.js +3 -0
package/lib/v2/cscse/tzgg.js +54 -0
package/lib/v2/dgjyw/index.js +67 -0
package/lib/v2/dgjyw/maintainer.js +3 -0
package/lib/v2/dgjyw/radar.js +13 -0
package/lib/v2/dgjyw/router.js +3 -0
package/lib/v2/domp4/detail.js +67 -0
package/lib/v2/domp4/latest.js +31 -0
package/lib/v2/domp4/maintainer.js +4 -0
package/lib/v2/domp4/radar.js +25 -0
package/lib/v2/domp4/router.js +4 -0
package/lib/v2/domp4/utils.js +72 -0
package/lib/v2/douyin/hashtag.js +5 -43
package/lib/v2/douyin/live.js +68 -0
package/lib/v2/douyin/maintainer.js +2 -0
package/lib/v2/douyin/radar.js +8 -0
package/lib/v2/douyin/router.js +1 -0
package/lib/v2/douyin/user.js +5 -43
package/lib/v2/douyin/utils.js +50 -0
package/lib/v2/economist/full.js +6 -4
package/lib/v2/elsevier/issue.js +62 -0
package/lib/v2/elsevier/journal.js +72 -0
package/lib/v2/elsevier/maintainer.js +2 -2
package/lib/v2/elsevier/radar.js +5 -5
package/lib/v2/elsevier/router.js +4 -2
package/lib/v2/elsevier/templates/description.art +5 -5
package/lib/v2/gov/beijing/jw/tzgg.js +58 -0
package/lib/v2/gov/customs/list.js +21 -17
package/lib/v2/gov/customs/utils.js +9 -5
package/lib/v2/gov/maintainer.js +3 -0
package/lib/v2/gov/nmpa/generic.js +76 -0
package/lib/v2/gov/pbc/goutongjiaoliu.js +20 -15
package/lib/v2/gov/pbc/tradeAnnouncement.js +18 -14
package/lib/v2/gov/radar.js +57 -27
package/lib/v2/gov/router.js +3 -0
package/lib/v2/gov/xuzhou/hrss.js +54 -0
package/lib/v2/hackertalk/index.js +23 -0
package/lib/v2/hackertalk/maintainer.js +3 -0
package/lib/v2/hackertalk/radar.js +13 -0
package/lib/v2/hackertalk/router.js +3 -0
package/lib/v2/hashnode/blog.js +62 -0
package/lib/v2/hashnode/maintainer.js +3 -0
package/lib/v2/hashnode/radar.js +13 -0
package/lib/v2/hashnode/router.js +3 -0
package/lib/v2/hashnode/templates/description.art +2 -0
package/lib/v2/hdu/cs/{index.js → notice.js} +1 -4
package/lib/v2/hdu/cs/pg.js +54 -0
package/lib/v2/hdu/maintainer.js +1 -0
package/lib/v2/hdu/radar.js +6 -0
package/lib/v2/hdu/router.js +2 -1
package/lib/v2/heu/gx/list.js +2 -2
package/lib/v2/heu/job/list.js +2 -2
package/lib/v2/heu/uae/list.js +2 -2
package/lib/v2/heu/yjsy/list.js +2 -2
package/lib/v2/hk01/channel.js +40 -0
package/lib/{routes → v2}/hk01/hot.js +2 -1
package/lib/v2/hk01/issue.js +41 -0
package/lib/v2/hk01/maintainer.js +7 -0
package/lib/v2/hk01/radar.js +37 -0
package/lib/v2/hk01/router.js +7 -0
package/lib/{routes → v2}/hk01/tag.js +2 -1
package/lib/{routes → v2}/hk01/zone.js +3 -2
package/lib/v2/hotukdeals/index.js +1 -1
package/lib/v2/ieee/journal.js +78 -0
package/lib/v2/ieee/maintainer.js +2 -2
package/lib/v2/ieee/radar.js +4 -4
package/lib/v2/ieee/{latestdate.js → recent.js} +34 -42
package/lib/v2/ieee/router.js +4 -2
package/lib/v2/ieee/templates/description.art +5 -5
package/lib/v2/ielts/index.js +7 -2
package/lib/v2/keep/maintainer.js +3 -0
package/lib/v2/keep/radar.js +13 -0
package/lib/v2/keep/router.js +3 -0
package/lib/v2/keep/templates/user.art +16 -0
package/lib/{routes → v2}/keep/user.js +14 -4
package/lib/v2/mdpi/journal.js +65 -0
package/lib/v2/mdpi/maintainer.js +3 -0
package/lib/v2/mdpi/radar.js +13 -0
package/lib/v2/mdpi/router.js +3 -0
package/lib/v2/mdpi/templates/description.art +12 -0
package/lib/v2/mindmeister/example.js +44 -0
package/lib/v2/mindmeister/maintainer.js +3 -0
package/lib/v2/mindmeister/radar.js +13 -0
package/lib/v2/mindmeister/router.js +3 -0
package/lib/v2/mindmeister/templates/image.art +1 -0
package/lib/v2/mirror/index.js +34 -0
package/lib/v2/mirror/maintainer.js +3 -0
package/lib/v2/mirror/radar.js +17 -0
package/lib/v2/mirror/router.js +3 -0
package/lib/v2/nbd/radar.js +2 -2
package/lib/v2/ncwu/maintainer.js +3 -0
package/lib/v2/ncwu/notice.js +27 -0
package/lib/v2/ncwu/radar.js +13 -0
package/lib/v2/ncwu/router.js +3 -0
package/lib/v2/netease/exclusive.js +148 -0
package/lib/v2/netease/maintainer.js +1 -0
package/lib/v2/netease/radar.js +8 -0
package/lib/v2/netease/router.js +1 -0
package/lib/v2/netease/templates/exclusive.art +11 -0
package/lib/v2/nju/rczp.js +34 -24
package/lib/v2/njust/utils.js +4 -0
package/lib/v2/nuaa/utils/pypasswaf.js +4 -5
package/lib/v2/nyaa/maintainer.js +4 -0
package/lib/v2/nyaa/radar.js +32 -0
package/lib/v2/nyaa/router.js +4 -0
package/lib/{routes → v2}/nyaa/search.js +4 -2
package/lib/v2/nytimes/daily_briefing_chinese.js +47 -41
package/lib/v2/nytimes/index.js +3 -3
package/lib/v2/nytimes/maintainer.js +1 -1
package/lib/v2/nytimes/templates/image.art +3 -0
package/lib/v2/nytimes/utils.js +7 -1
package/lib/v2/oceanengine/arithmeticIndex.js +145 -0
package/lib/v2/oceanengine/maintainer.js +3 -0
package/lib/v2/oceanengine/radar.js +13 -0
package/lib/v2/oceanengine/router.js +3 -0
package/lib/v2/oceanengine/templates/content.art +14 -0
package/lib/v2/openwrt/maintainer.js +3 -0
package/lib/v2/openwrt/radar.js +13 -0
package/lib/v2/openwrt/releases.js +25 -0
package/lib/v2/openwrt/router.js +3 -0
package/lib/v2/pincong/utils.js +7 -1
package/lib/v2/pku/maintainer.js +3 -0
package/lib/v2/pku/radar.js +20 -0
package/lib/v2/pku/router.js +3 -0
package/lib/v2/pku/ss/admission.js +53 -0
package/lib/v2/pku/ss/notice.js +53 -0
package/lib/v2/pku/ss/pg_admin.js +53 -0
package/lib/v2/prestige-av/maintainer.js +3 -0
package/lib/v2/prestige-av/radar.js +18 -0
package/lib/v2/prestige-av/router.js +3 -0
package/lib/{routes → v2}/prestige-av/series.js +8 -5
package/lib/v2/qq/kg/cache.js +28 -0
package/lib/v2/qq/kg/reply.js +21 -0
package/lib/v2/qq/kg/user.js +50 -0
package/lib/v2/qq/maintainer.js +2 -0
package/lib/v2/qq/radar.js +14 -0
package/lib/v2/qq/router.js +2 -0
package/lib/v2/samsung/maintainer.js +3 -0
package/lib/v2/samsung/radar.js +13 -0
package/lib/v2/samsung/research/blog.js +49 -0
package/lib/v2/samsung/router.js +3 -0
package/lib/v2/{jasa/latest.js → scitation/journal.js} +24 -27
package/lib/v2/scitation/maintainer.js +4 -0
package/lib/v2/scitation/radar.js +19 -0
package/lib/v2/scitation/router.js +4 -0
package/lib/v2/scitation/section.js +90 -0
package/lib/v2/scitation/templates/description.art +11 -0
package/lib/v2/shuiguopai/index.js +76 -0
package/lib/v2/shuiguopai/maintainer.js +3 -0
package/lib/v2/shuiguopai/radar.js +13 -0
package/lib/v2/shuiguopai/router.js +3 -0
package/lib/v2/shuiguopai/templates/description.art +6 -0
package/lib/v2/springer/journal.js +72 -0
package/lib/v2/springer/maintainer.js +3 -0
package/lib/v2/springer/radar.js +13 -0
package/lib/v2/springer/router.js +3 -0
package/lib/v2/springer/templates/description.art +12 -0
package/lib/v2/supchina/index.js +67 -0
package/lib/v2/supchina/maintainer.js +4 -0
package/lib/v2/supchina/podcasts.js +66 -0
package/lib/v2/supchina/radar.js +19 -0
package/lib/v2/supchina/router.js +4 -0
package/lib/v2/techflow520/index.js +34 -0
package/lib/v2/techflow520/maintainer.js +4 -0
package/lib/v2/techflow520/newsflash.js +32 -0
package/lib/v2/techflow520/radar.js +19 -0
package/lib/v2/techflow520/router.js +4 -0
package/lib/v2/tencent/cloud/column.js +51 -0
package/lib/v2/tencent/maintainer.js +1 -0
package/lib/v2/tencent/radar.js +11 -0
package/lib/v2/tencent/router.js +1 -0
package/lib/v2/test/index.js +16 -1
package/lib/v2/tiktok/user.js +28 -38
package/lib/v2/trending/allTrending.js +169 -0
package/lib/v2/trending/maintainer.js +3 -0
package/lib/v2/trending/radar.js +35 -0
package/lib/v2/trending/router.js +3 -0
package/lib/v2/trending/templates/content.art +24 -0
package/lib/v2/u3c3/index.js +59 -0
package/lib/v2/u3c3/maintainer.js +3 -0
package/lib/v2/u3c3/radar.js +29 -0
package/lib/v2/u3c3/router.js +3 -0
package/lib/v2/uraaka-joshi/uraaka-joshi-user.js +7 -1
package/lib/v2/uraaka-joshi/uraaka-joshi.js +8 -1
package/lib/v2/vlive/index.js +71 -0
package/lib/v2/vlive/maintainer.js +3 -0
package/lib/v2/vlive/radar.js +13 -0
package/lib/v2/vlive/router.js +3 -0
package/lib/v2/vlive/templates/post.art +8 -0
package/lib/v2/vlive/templates/video.art +3 -0
package/lib/v2/xiaohongshu/util.js +6 -6
package/lib/v2/xjtu/2yuan/news.js +61 -0
package/lib/v2/xjtu/dean.js +83 -0
package/lib/{routes/universities → v2}/xjtu/ee.js +5 -5
package/lib/{routes/universities → v2}/xjtu/gs/tzgg.js +6 -6
package/lib/v2/xjtu/international.js +51 -0
package/lib/v2/xjtu/job.js +81 -0
package/lib/v2/xjtu/maintainer.js +9 -0
package/lib/v2/xjtu/radar.js +53 -0
package/lib/v2/xjtu/router.js +9 -0
package/lib/v2/xjtu/std.js +58 -0
package/lib/v2/xjtu/templates/attachments.art +3 -0
package/lib/v2/xjtu/templates/std.art +4 -0
package/lib/v2/xueqiu/column.js +42 -0
package/lib/v2/xueqiu/maintainer.js +1 -0
package/lib/v2/xueqiu/radar.js +6 -0
package/lib/v2/xueqiu/router.js +1 -0
package/lib/v2/zhangyoubao/index.js +42 -0
package/lib/v2/zhangyoubao/maintainer.js +3 -0
package/lib/v2/zhangyoubao/radar.js +13 -0
package/lib/v2/zhangyoubao/router.js +3 -0
package/lib/v2/zhangyoubao/templates/article.art +1 -0
package/package.json +20 -19
package/lib/routes/acfun/video.js +0 -44
package/lib/routes/apnews/topics.js +0 -43
package/lib/routes/dgjyw/index.js +0 -66
package/lib/routes/hk01/channel.js +0 -22
package/lib/routes/hk01/issue.js +0 -22
package/lib/routes/mirror/entries.js +0 -80
package/lib/routes/universities/ncwu/notice.js +0 -42
package/lib/routes/universities/xjtu/dean.js +0 -86
package/lib/routes/universities/xjtu/international.js +0 -53
package/lib/routes/universities/xjtu/job.js +0 -50
package/lib/v2/elsevier/latest.js +0 -87
package/lib/v2/elsevier/volume.js +0 -75
package/lib/v2/ieee/latestvol.js +0 -87
package/lib/v2/jasa/maintainer.js +0 -4
package/lib/v2/jasa/radar.js +0 -19
package/lib/v2/jasa/router.js +0 -4
package/lib/v2/jasa/section.js +0 -95
package/lib/v2/jasa/templates/description.art +0 -11

package/lib/config.js CHANGED Viewed

@@ -91,6 +91,10 @@ const calculateValue = () => {
             includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','),
             excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','),
         },
+        feature: {
+            allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true',
+            filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
+        },
         suffix: envs.SUFFIX,
         titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150,

package/lib/middleware/anti-hotlink.js CHANGED Viewed

@@ -1,6 +1,12 @@
 const config = require('@/config').value;
 const cheerio = require('cheerio');
 const logger = require('@/utils/logger');
+const path = require('path');
+const { art } = require('@/utils/render');
+const templateRegex = /\$\{([^{}]+)}/g;
+const allowedUrlProperties = ['hash', 'host', 'hostname', 'href', 'origin', 'password', 'pathname', 'port', 'protocol', 'search', 'searchParams', 'username'];
+const IframeWrapperTemplate = path.join(__dirname, 'templates/iframe.art');
 // match path or sub-path
 const matchPath = (path, paths) => {
@@ -19,9 +25,16 @@ const filterPath = (path) => {
     return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude));
 };
-const interpolate = (str, obj) => str.replace(/\${([^}]+)}/g, (_, prop) => obj[prop]);
-// I don't want to keep another regex and
-// URL will be the standard way to parse URL
+const interpolate = (str, obj) =>
+    str.replace(templateRegex, (_, prop) => {
+        let needEncode = false;
+        if (prop.endsWith('_ue')) {
+            // url encode
+            prop = prop.slice(0, -3);
+            needEncode = true;
+        }
+        return needEncode ? encodeURIComponent(obj[prop]) : obj[prop];
+    });
 const parseUrl = (str) => {
     let url;
     try {
@@ -32,48 +45,103 @@ const parseUrl = (str) => {
     return url;
 };
-const replaceUrls = (body, template) => {
-    // const $ = cheerio.load(body, { decodeEntities: false, xmlMode: true });
-    // `<br><img><hr><video><source>abc</video>` => `<br><img><hr><video><source>abc</source></video></hr></img></br>`
-    // so awful...
-    // "In HTML, using a closing tag on an empty element is usually invalid."
-    // https://developer.mozilla.org/en-US/docs/Glossary/Empty_element
-    // I guess it is just a workaround to drop `<html><head></head><body>`, so this is what we exactly need:
-    const $ = cheerio.load(body, null, false);
-    $('img').each(function () {
-        const old_src = $(this).attr('src');
-        const url = parseUrl(old_src);
-        if (url) {
-            const new_src = interpolate(template, url);
-            $(this).attr('src', new_src);
+const replaceUrls = ($, selector, template, attribute = 'src') => {
+    $(selector).each(function () {
+        const old_src = $(this).attr(attribute);
+        if (old_src) {
+            const url = parseUrl(old_src);
+            if (url) {
+                // Cheerio will do the right thing to prohibit XSS.
+                $(this).attr(attribute, interpolate(template, url));
+            }
         }
     });
+};
+const wrapWithIframe = ($, selector) => {
+    $(selector).each((_, elem) => {
+        elem = $(elem);
+        elem.replaceWith(art(IframeWrapperTemplate, { content: elem.toString() }));
+    });
+};
-    return $.root().html();
+const process = (html, image_hotlink_template, multimedia_hotlink_template, wrap_multimedia_in_iframe) => {
+    const $ = cheerio.load(html, undefined, false);
+    if (image_hotlink_template) {
+        replaceUrls($, 'img, picture > source', image_hotlink_template);
+        replaceUrls($, 'video[poster]', image_hotlink_template, 'poster');
+    }
+    if (multimedia_hotlink_template) {
+        replaceUrls($, 'video, video > source, audio, audio > source', multimedia_hotlink_template);
+        if (!image_hotlink_template) {
+            replaceUrls($, 'video[poster]', multimedia_hotlink_template, 'poster');
+        }
+    }
+    if (wrap_multimedia_in_iframe) {
+        wrapWithIframe($, 'video, audio');
+    }
+    return $.html();
+};
+const validateTemplate = (template) => {
+    if (!template) {
+        return;
+    }
+    [...template.matchAll(templateRegex)].forEach((match) => {
+        const prop = match[1].endsWith('_ue') ? match[1].slice(0, -3) : match[1];
+        if (!allowedUrlProperties.includes(prop)) {
+            throw new Error(`Invalid URL property: ${prop}`);
+        }
+    });
 };
 module.exports = async (ctx, next) => {
     await next();
-    const template = config.hotlink.template;
+    let image_hotlink_template = undefined;
+    let multimedia_hotlink_template = undefined;
+    const shouldWrapInIframe = ctx.query.wrap_multimedia_in_iframe === '1';
-    if (!template || !filterPath(ctx.request.path)) {
+    // Read params if enabled
+    if (config.feature.allow_user_hotlink_template) {
+        // By default, the config turns these features off. Set corresponding config to
+        // true to turn this feature on.
+        // A risk is that the media URLs will be replaced by user-supplied templates,
+        // so a user could literally take the control of "where are the media from",
+        // but only in their personal-use feed URL.
+        multimedia_hotlink_template = ctx.query.multimedia_hotlink_template;
+        image_hotlink_template = ctx.query.image_hotlink_template;
+    }
+    // Force config hotlink template on conflict
+    if (config.hotlink.template) {
+        if (!filterPath(ctx.request.path)) {
+            image_hotlink_template = undefined;
+        } else {
+            image_hotlink_template = config.hotlink.template;
+        }
+    }
+    if (!image_hotlink_template && !multimedia_hotlink_template && !shouldWrapInIframe) {
         return;
     }
+    validateTemplate(image_hotlink_template);
+    validateTemplate(multimedia_hotlink_template);
     // Assume that only description include image link
     // and here we will only check them in description.
     // Use Cheerio to load the description as html and filter all
     // image link
     if (ctx.state.data) {
         if (ctx.state.data.description) {
-            ctx.state.data.description = replaceUrls(ctx.state.data.description, template);
+            ctx.state.data.description = process(ctx.state.data.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
         }
         ctx.state.data.item &&
             ctx.state.data.item.forEach((item) => {
                 if (item.description) {
-                    item.description = replaceUrls(item.description, template);
+                    item.description = process(item.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe);
                 }
             });
     }

package/lib/middleware/parameter.js CHANGED Viewed

@@ -3,9 +3,26 @@ const cheerio = require('cheerio');
 const { simplecc } = require('simplecc-wasm');
 const got = require('@/utils/got');
 const config = require('@/config').value;
+const RE2 = require('re2');
 let mercury_parser;
+const resolveRelativeLink = ($, elem, attr, baseUrl) => {
+    const $elem = $(elem);
+    if (baseUrl) {
+        try {
+            const oldAttr = $elem.attr(attr);
+            if (oldAttr) {
+                // e.g. <video><source src="https://example.com"></video> should leave <video> unchanged
+                $elem.attr(attr, new URL(oldAttr, baseUrl).href);
+            }
+        } catch (e) {
+            // no-empty
+        }
+    }
+};
 module.exports = async (ctx, next) => {
     await next();
@@ -76,50 +93,49 @@ module.exports = async (ctx, next) => {
                 $('script').remove();
-                $('a').each((_, ele) => {
-                    const $ele = $(ele);
-                    // absolute link
-                    if (baseUrl) {
-                        try {
-                            $ele.attr('href', new URL($ele.attr('href'), baseUrl).href);
-                        } catch (e) {
-                            // no-empty
-                        }
-                    }
-                });
                 $('img').each((_, ele) => {
                     const $ele = $(ele);
                     // fix lazyload
                     if (!$ele.attr('src')) {
-                        for (const key in ele.attribs) {
-                            const value = ele.attribs[key].trim();
-                            if (['.gif', '.png', '.jpg', '.webp'].some((suffix) => value.includes(suffix))) {
-                                $ele.attr('src', value);
-                                break;
+                        const lazySrc = $ele.attr('data-src') || $ele.attr('data-original');
+                        if (lazySrc) {
+                            $ele.attr('src', lazySrc);
+                        } else {
+                            for (const key in ele.attribs) {
+                                const value = ele.attribs[key].trim();
+                                if (['.gif', '.png', '.jpg', '.webp'].some((suffix) => value.includes(suffix))) {
+                                    $ele.attr('src', value);
+                                    break;
+                                }
                             }
                         }
                     }
-                    // absolute link
-                    if (baseUrl) {
-                        try {
-                            $ele.attr('src', new URL($ele.attr('src'), baseUrl).href);
-                        } catch (e) {
-                            // no-empty
-                        }
-                    }
-                    // referrerpolicy
-                    $ele.attr('referrerpolicy', 'no-referrer');
                     // redundant attributes
                     ['onclick', 'onerror', 'onload'].forEach((e) => {
                         $ele.removeAttr(e);
                     });
                 });
+                // resolve relative link & fix referrer policy
+                // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy
+                // https://www.w3schools.com/tags/att_href.asp
+                $('a, area').each((_, elem) => {
+                    resolveRelativeLink($, elem, 'href', baseUrl);
+                    // $(elem).attr('rel', 'noreferrer');  // currently no such a need
+                });
+                // https://www.w3schools.com/tags/att_src.asp
+                $('img, video, audio, source, iframe, embed, track').each((_, elem) => {
+                    resolveRelativeLink($, elem, 'src', baseUrl);
+                });
+                $('video[poster]').each((_, elem) => {
+                    resolveRelativeLink($, elem, 'poster', baseUrl);
+                });
+                $('img, iframe').each((_, elem) => {
+                    $(elem).attr('referrerpolicy', 'no-referrer');
+                });
                 item.description = $('body').html() + '' + (config.suffix || '');
             }
             return item;
@@ -136,10 +152,25 @@ module.exports = async (ctx, next) => {
             // filter
             const makeRegex = (string) => {
                 // default: case_senstivie = true
+                const engine = config.feature.filter_regex_engine;
                 if (ctx.query.filter_case_sensitive === 'false') {
-                    return new RegExp(string, 'i');
+                    switch (engine) {
+                        case 'regexp':
+                            return new RegExp(string, 'i');
+                        case 're2':
+                            return new RE2(string, 'i');
+                        default:
+                            throw Error(`Invalid Engine Value: ${engine}, please check your config.`);
+                    }
                 } else {
-                    return new RegExp(string);
+                    switch (engine) {
+                        case 'regexp':
+                            return new RegExp(string);
+                        case 're2':
+                            return new RE2(string);
+                        default:
+                            throw Error(`Invalid Engine Value: ${engine}, please check your config.`);
+                    }
                 }
             };
@@ -272,6 +303,8 @@ module.exports = async (ctx, next) => {
                     throw Error(`Invalid parameter <code>brief=${ctx.query.brief}</code>. Please check the doc https://docs.rsshub.app/parameter.html#shu-chu-jian-xun`);
                 }
             }
+            // some parameters are processed in `anti-hotlink.js`
         }
     }
 };

package/lib/middleware/templates/iframe.art ADDED Viewed

@@ -0,0 +1,14 @@
+<iframe referrerpolicy="no-referrer" width=100% height=150vh frameborder=0 marginheight=0 marginwidth=0
+style="border:0; margin:0; padding:0; width:100%; height:150vh;"
+srcdoc="
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta name=&quot;referrer&quot; content=&quot;no-referrer&quot;>
+  </head>
+  <body>
+    {{ content }}
+  </body>
+</html>
+">
+</iframe>

package/lib/radar-rules.js CHANGED Viewed

@@ -1135,17 +1135,6 @@ module.exports = {
             },
         ],
     },
-    'apnews.com': {
-        _name: 'AP News',
-        '.': [
-            {
-                title: '话题',
-                docs: 'https://docs.rsshub.app/traditional-media.html#ap-news',
-                source: '/:topic',
-                target: '/apnews/topics/:topic',
-            },
-        ],
-    },
     'csc.edu.cn': {
         _name: '国家留学网',
         www: [
@@ -1589,41 +1578,6 @@ module.exports = {
             },
         ],
     },
-    'hk01.com': {
-        _name: '香港01',
-        www: [
-            {
-                title: '最 Hit',
-                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
-                source: ['/hot', '/'],
-                target: '/hk01/hot',
-            },
-            {
-                title: 'zone',
-                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
-                source: '/zone/:id/:title',
-                target: '/hk01/zone/:id',
-            },
-            {
-                title: 'channel',
-                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
-                source: '/channel/:id/:title',
-                target: '/hk01/channel/:id',
-            },
-            {
-                title: 'issue',
-                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
-                source: '/issue/:id/:title',
-                target: '/hk01/issue/:id',
-            },
-            {
-                title: 'tag',
-                docs: 'https://docs.rsshub.app/traditional-media.html#xiang-gang-01',
-                source: '/tag/:id/:title',
-                target: '/hk01/tag/:id',
-            },
-        ],
-    },
     'popiapp.cn': {
         _name: 'Popi 提问箱',
         www: [
@@ -1656,23 +1610,6 @@ module.exports = {
             },
         ],
     },
-    'acfun.cn': {
-        _name: 'AcFun',
-        www: [
-            {
-                tilte: '番剧',
-                docs: 'https://docs.rsshub.app/anime.html#acfun-fan-ju',
-                source: '/bangumi/:id',
-                target: (params) => `/acfun/bangumi/${params.id.replace('aa', '')}`,
-            },
-            {
-                title: '用户投稿',
-                docs: 'https://docs.rsshub.app/anime.html#acfun-yong-hu-tou-gao',
-                source: '/u/:id',
-                target: '/acfun/user/video/:id',
-            },
-        ],
-    },
     'jjmhw.cc': {
         _name: '漫小肆',
         www: [
@@ -1981,35 +1918,6 @@ module.exports = {
             },
         ],
     },
-    'gcores.com': {
-        _name: '机核网',
-        www: [
-            {
-                title: '资讯',
-                docs: 'https://docs.rsshub.app/program-update.html#eagle',
-                source: '/news',
-                target: '/gcores/category/news',
-            },
-            {
-                title: '视频',
-                docs: 'https://docs.rsshub.app/program-update.html#eagle',
-                source: '/videos',
-                target: '/gcores/category/videos',
-            },
-            {
-                title: '电台',
-                docs: 'https://docs.rsshub.app/program-update.html#eagle',
-                source: '/radios',
-                target: '/gcores/category/radios',
-            },
-            {
-                title: '文章',
-                docs: 'https://docs.rsshub.app/program-update.html#eagle',
-                source: '/articles',
-                target: '/gcores/category/articles',
-            },
-        ],
-    },
     'bgm.tv': {
         _name: 'Bangumi',
         '.': [

package/lib/router.js CHANGED Viewed

@@ -228,8 +228,8 @@ router.get('/pornhub/:language?/users/:username', lazyloadRouteHandler('./routes
 router.get('/pornhub/:language?/model/:username/:sort?', lazyloadRouteHandler('./routes/pornhub/model'));
 router.get('/pornhub/:language?/pornstar/:username/:sort?', lazyloadRouteHandler('./routes/pornhub/pornstar'));
-// Prestige
-router.get('/prestige-av/series/:mid/:sort?', lazyloadRouteHandler('./routes/prestige-av/series'));
+// Prestige migrated to v2
+// router.get('/prestige-av/series/:mid/:sort?', lazyloadRouteHandler('./routes/prestige-av/series'));
 // yande.re
 router.get('/yande.re/post/popular_recent', lazyloadRouteHandler('./routes/yande.re/post_popular_recent'));
@@ -336,7 +336,7 @@ router.get('/miui/:device/:type?/:region?', lazyloadRouteHandler('./routes/mi/mi
 router.get('/mi/bbs/board/:boardId', lazyloadRouteHandler('./routes/mi/board'));
 // Keep
-router.get('/keep/user/:id', lazyloadRouteHandler('./routes/keep/user'));
+// router.get('/keep/user/:id', lazyloadRouteHandler('./routes/keep/user'));
 // 起点
 router.get('/qidian/chapter/:id', lazyloadRouteHandler('./routes/qidian/chapter'));
@@ -759,7 +759,7 @@ router.get('/upc/main/:type?', lazyloadRouteHandler('./routes/universities/upc/m
 router.get('/upc/jsj/:type?', lazyloadRouteHandler('./routes/universities/upc/jsj'));
 // 华北水利水电大学
-router.get('/ncwu/notice', lazyloadRouteHandler('./routes/universities/ncwu/notice'));
+// router.get('/ncwu/notice', lazyloadRouteHandler('./routes/universities/ncwu/notice'));
 // 太原师范学院
 router.get('/tynu', lazyloadRouteHandler('./routes/universities/tynu/tynu'));
@@ -939,9 +939,9 @@ router.get('/nhentai/:key/:keyword/:mode?', lazyloadRouteHandler('./routes/nhent
 // 龙腾网
 router.get('/ltaaa/:category?', lazyloadRouteHandler('./routes/ltaaa/index'));
-// AcFun
-router.get('/acfun/bangumi/:id', lazyloadRouteHandler('./routes/acfun/bangumi'));
-router.get('/acfun/user/video/:uid', lazyloadRouteHandler('./routes/acfun/video'));
+// AcFun migrated to v2
+// router.get('/acfun/bangumi/:id', lazyloadRouteHandler('./routes/acfun/bangumi'));
+// router.get('/acfun/user/video/:uid', lazyloadRouteHandler('./routes/acfun/video'));
 // Auto Trader
 router.get('/autotrader/:query', lazyloadRouteHandler('./routes/autotrader'));
@@ -2201,11 +2201,11 @@ router.get('/mcbbs/post/:tid/:authorid?', lazyloadRouteHandler('./routes/mcbbs/p
 router.get('/pocket/trending', lazyloadRouteHandler('./routes/pocket/trending'));
 // HK01
-router.get('/hk01/zone/:id', lazyloadRouteHandler('./routes/hk01/zone'));
-router.get('/hk01/channel/:id', lazyloadRouteHandler('./routes/hk01/channel'));
-router.get('/hk01/issue/:id', lazyloadRouteHandler('./routes/hk01/issue'));
-router.get('/hk01/tag/:id', lazyloadRouteHandler('./routes/hk01/tag'));
-router.get('/hk01/hot', lazyloadRouteHandler('./routes/hk01/hot'));
+// router.get('/hk01/zone/:id', lazyloadRouteHandler('./routes/hk01/zone'));
+// router.get('/hk01/channel/:id', lazyloadRouteHandler('./routes/hk01/channel'));
+// router.get('/hk01/issue/:id', lazyloadRouteHandler('./routes/hk01/issue'));
+// router.get('/hk01/tag/:id', lazyloadRouteHandler('./routes/hk01/tag'));
+// router.get('/hk01/hot', lazyloadRouteHandler('./routes/hk01/hot'));
 // 码农周刊
 router.get('/manong-weekly', lazyloadRouteHandler('./routes/manong-weekly/issues'));
@@ -2265,8 +2265,8 @@ router.get('/transferwise/pair/:source/:target', lazyloadRouteHandler('./routes/
 // chocolatey
 router.get('/chocolatey/software/:name?', lazyloadRouteHandler('./routes/chocolatey/software'));
-// Nyaa
-router.get('/nyaa/search/:query?', lazyloadRouteHandler('./routes/nyaa/search'));
+// Nyaa migrated to v2
+// router.get('/nyaa/search/:query?', lazyloadRouteHandler('./routes/nyaa/search'));
 // 片源网 migrated to v2
 // router.get('/pianyuan/index/:media?', lazyloadRouteHandler('./routes/pianyuan/app'));
@@ -2364,7 +2364,7 @@ router.get('/xinquji/today/internal', lazyloadRouteHandler('./routes/xinquji/int
 router.get('/gbcc/trust', lazyloadRouteHandler('./routes/gbcc/trust'));
 // Associated Press
-router.get('/apnews/topics/:topic', lazyloadRouteHandler('./routes/apnews/topics'));
+// router.get('/apnews/topics/:topic', lazyloadRouteHandler('./routes/apnews/topics'));
 // CBC
 router.get('/cbc/topics/:topic?', lazyloadRouteHandler('./routes/cbc/topics'));
@@ -2492,11 +2492,11 @@ router.get('/wolley/user/:id', lazyloadRouteHandler('./routes/wolley/user'));
 router.get('/wolley/host/:host', lazyloadRouteHandler('./routes/wolley/host'));
 // 西安交大
-router.get('/xjtu/gs/tzgg', lazyloadRouteHandler('./routes/universities/xjtu/gs/tzgg'));
-router.get('/xjtu/dean/:subpath+', lazyloadRouteHandler('./routes/universities/xjtu/dean'));
-router.get('/xjtu/international/:subpath+', lazyloadRouteHandler('./routes/universities/xjtu/international'));
-router.get('/xjtu/job/:subpath?', lazyloadRouteHandler('./routes/universities/xjtu/job'));
-router.get('/xjtu/ee/:id?', lazyloadRouteHandler('./routes/universities/xjtu/ee'));
+// router.get('/xjtu/gs/tzgg', lazyloadRouteHandler('./routes/universities/xjtu/gs/tzgg'));
+// router.get('/xjtu/dean/:subpath+', lazyloadRouteHandler('./routes/universities/xjtu/dean'));
+// router.get('/xjtu/international/:subpath+', lazyloadRouteHandler('./routes/universities/xjtu/international'));
+// router.get('/xjtu/job/:subpath?', lazyloadRouteHandler('./routes/universities/xjtu/job'));
+// router.get('/xjtu/ee/:id?', lazyloadRouteHandler('./routes/universities/xjtu/ee'));
 // booksource
 router.get('/booksource', lazyloadRouteHandler('./routes/booksource/index'));
@@ -2863,7 +2863,7 @@ router.get('/law/jctd', lazyloadRouteHandler('./routes/law/jctd'));
 router.get('/samsungmembers/latest', lazyloadRouteHandler('./routes/samsungmembers/latest'));
 // 东莞教研网
-router.get('/dgjyw/:type', lazyloadRouteHandler('./routes/dgjyw/index'));
+// router.get('/dgjyw/:type', lazyloadRouteHandler('./routes/dgjyw/index'));
 // 中国信息通信研究院
 router.get('/gov/caict/bps', lazyloadRouteHandler('./routes/gov/caict/bps'));
@@ -4065,7 +4065,7 @@ router.get('/secrss/author/:author?', lazyloadRouteHandler('./routes/secrss/auth
 router.get('/fashionnetwork/headline/:country?', lazyloadRouteHandler('./routes/fashionnetwork/headline.js'));
 // mirror.xyz
-router.get('/mirror/:id', lazyloadRouteHandler('./routes/mirror/entries'));
+// router.get('/mirror/:id', lazyloadRouteHandler('./routes/mirror/entries'));
 // KBS migrated to v2
 // router.get('/kbs/today/:language?', lazyloadRouteHandler('./routes/kbs/today'));

package/lib/routes/agefans/update.js CHANGED Viewed

@@ -2,7 +2,7 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 module.exports = async (ctx) => {
-    const rootUrl = 'https://www.agefans.cc';
+    const rootUrl = 'https://www.agemys.com';
     const currentUrl = `${rootUrl}/update`;
     const response = await got({
         method: 'get',

package/lib/routes/novel/biqugeinfo.js CHANGED Viewed

@@ -35,7 +35,7 @@ module.exports = async (ctx) => {
     const items = [];
     for await (const item of asyncPool(3, chapter_item, (item) =>
-        ctx.cache.tryGet(item, async () => {
+        ctx.cache.tryGet(item.link.href, async () => {
             const response = await got({
                 method: 'get',
                 url: item.link,
@@ -61,7 +61,7 @@ module.exports = async (ctx) => {
     ctx.state.data = {
         title: `笔趣阁 ${title}`,
-        link: pageUrl,
+        link: pageUrl.href,
         image: cover_url,
         description,
         item: items,

package/lib/routes/universities/uestc/sice.js CHANGED Viewed

@@ -1,40 +1,18 @@
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const dayjs = require('dayjs');
 module.exports = async (ctx) => {
     const baseIndexUrl = 'https://www.sice.uestc.edu.cn/index.htm';
     const host = 'https://www.sice.uestc.edu.cn/';
-    const browser = await require('@/utils/puppeteer')();
+    const browser = await require('@/utils/puppeteer')({ stealth: true });
     const page = await browser.newPage();
-    // 浏览器伪装
-    await page.evaluateOnNewDocument(() => {
-        // 在每个新页面打开前执行以下脚本
-        const newProto = navigator.__proto__;
-        delete newProto.webdriver; // 删除navigator.webdriver字段
-        navigator.__proto__ = newProto;
-        window.chrome = {}; // 添加window.chrome字段，为增加真实性还需向内部填充一些值
-        window.chrome.app = { InstallState: 'hehe', RunningState: 'haha', getDetails: 'xixi', getIsInstalled: 'ohno' };
-        window.chrome.csi = function () {};
-        window.chrome.loadTimes = function () {};
-        window.chrome.runtime = function () {};
-        Object.defineProperty(navigator, 'userAgent', {
-            // userAgent在无头模式下有headless字样，所以需覆写
-            get: () => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.113 Safari/537.36',
-        });
-        Object.defineProperty(navigator, 'plugins', {
-            // 伪装真实的插件信息
-            get: () => [{ description: 'Portable Document Format', filename: 'internal-pdf-viewer', length: 1, name: 'Chrome PDF Plugin' }],
-        });
-        Object.defineProperty(navigator, 'languages', {
-            // 添加语言
-            get: () => ['zh-CN', 'zh', 'en'],
-        });
-        const originalQuery = window.navigator.permissions.query; // notification伪装
-        window.navigator.permissions.query = (parameters) => (parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters));
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
     });
     await page.goto(baseIndexUrl, {
-        waitUntil: 'networkidle0',
+        waitUntil: 'networkidle2',
     });
     const content = await page.content();
     await browser.close();
@@ -43,14 +21,15 @@ module.exports = async (ctx) => {
     const out = $('.notice p')
         .map((index, item) => {
             item = $(item);
-            let date = new Date(new Date().getFullYear() + '-' + item.find('a.date').text());
-            if (new Date() < date) {
-                date = new Date(new Date().getFullYear() - 1 + '-' + item.find('a.date').text());
+            const now = dayjs();
+            let date = dayjs(now.year() + '-' + item.find('a.date').text());
+            if (now < date) {
+                date = dayjs(now.year() - 1 + '-' + item.find('a.date').text());
             }
             return {
                 title: item.find('a[href]').text(),
                 link: host + item.find('a[href]').attr('href'),
-                pubDate: date,
+                pubDate: parseDate(date),
             };
         })
         .get();