rsshub 1.0.0-master.fe4f650 → 1.0.0-master.fe560d2

package/README.md

@@ -6,9 +6,9 @@
 > 🍰 Everything is RSSible
 
 [![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?logo=telegram&style=flat-square)](https://t.me/rsshub)
-[![npm publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/publish/master?label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
-[![docker publish](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/%5Bdocker%5D%20CI%20for%20releases/master?label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
-[![test](https://img.shields.io/github/workflow/status/DIYgod/RSSHub/test/master?label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
+[![npm publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/npm-publish.yml?branch=master&label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub)
+[![docker publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/docker-release.yml?branch=master&label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub)
+[![test](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/test.yml?branch=master&label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster)
 [![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square&logo=codecov)](https://app.codecov.io/gh/DIYgod/RSSHub/branch/master)
 [![Foresight Docs](https://api-public.service.runforesight.com/api/v1/badge/success?repoId=5da20e56-cecf-4b05-9bcd-cab097fb2dca)](https://docs.runforesight.com/)
 

package/lib/config.js

@@ -225,7 +225,7 @@ const calculateValue = () => {
             bypassCdn: envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false',
             bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net',
             bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query',
-            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.cat',
+            imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.re',
         },
         pkubbs: {
             cookie: envs.PKUBBS_COOKIE,

package/lib/middleware/parameter.js

@@ -181,7 +181,7 @@ module.exports = async (ctx, next) => {
                     const title = item.title || '';
                     const description = item.description || title;
                     const author = item.author || '';
-                    const category = item.category || [];
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
                     const isFilter =
                         title.match(makeRegex(ctx.query.filter)) || description.match(makeRegex(ctx.query.filter)) || author.match(makeRegex(ctx.query.filter)) || category.some((c) => c.match(makeRegex(ctx.query.filter)));
                     return isFilter;
@@ -194,7 +194,7 @@ module.exports = async (ctx, next) => {
                     const title = item.title || '';
                     const description = item.description || title;
                     const author = item.author || '';
-                    const category = item.category || [];
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
                     let isFilter = true;
                     ctx.query.filter_title && (isFilter = title.match(makeRegex(ctx.query.filter_title)));
                     ctx.query.filter_description && (isFilter = isFilter && description.match(makeRegex(ctx.query.filter_description)));
@@ -213,7 +213,7 @@ module.exports = async (ctx, next) => {
                     const title = item.title;
                     const description = item.description || title;
                     const author = item.author || '';
-                    const category = item.category || [];
+                    const category = item.category ? (Array.isArray(item.category) ? item.category : [item.category]) : [];
                     let isFilter = true;
                     ctx.query.filterout_title && (isFilter = !title.match(makeRegex(ctx.query.filterout_title)));
                     ctx.query.filterout_description && (isFilter = isFilter && !description.match(makeRegex(ctx.query.filterout_description)));

package/lib/radar-rules.js

@@ -1,50 +1,4 @@
 module.exports = {
-    'weibo.com': {
-        _name: '微博',
-        '.': [
-            {
-                title: '博主',
-                docs: 'https://docs.rsshub.app/social-media.html#wei-bo',
-                source: ['/u/:id', '/:id'],
-                target: (params, url, document) => {
-                    let uid = document?.documentElement.innerHTML.match(/\$CONFIG\['oid']='(\d+)'/)?.[1];
-                    if (!uid && !isNaN(params.id)) {
-                        uid = params.id;
-                    }
-                    return uid ? `/weibo/user/${uid}` : '';
-                },
-            },
-            {
-                title: '关键词',
-                docs: 'https://docs.rsshub.app/social-media.html#wei-bo',
-            },
-            {
-                title: '超话',
-                docs: 'https://docs.rsshub.app/social-media.html#wei-bo',
-                source: '/p/:id/super_index',
-                target: '/weibo/super_index/:id',
-            },
-        ],
-        s: [
-            {
-                title: '热搜榜',
-                docs: 'https://docs.rsshub.app/social-media.html#wei-bo',
-                source: '/top/summary',
-                target: '/weibo/search/hot',
-            },
-        ],
-    },
-    'weibo.cn': {
-        _name: '微博',
-        m: [
-            {
-                title: '博主',
-                docs: 'https://docs.rsshub.app/social-media.html#wei-bo',
-                source: ['/u/:uid', '/profile/:uid'],
-                target: '/weibo/user/:uid',
-            },
-        ],
-    },
     'ximalaya.com': {
         _name: '喜马拉雅',
         '.': [

package/lib/router.js

@@ -47,12 +47,12 @@ router.get('/benedictevans', lazyloadRouteHandler('./routes/benedictevans/recent
 // router.get('/twreporter/photography', lazyloadRouteHandler('./routes/twreporter/photography'));
 // router.get('/twreporter/category/:cid', lazyloadRouteHandler('./routes/twreporter/category'));
 
-// 微博
-router.get('/weibo/user/:uid/:routeParams?', lazyloadRouteHandler('./routes/weibo/user'));
-router.get('/weibo/keyword/:keyword/:routeParams?', lazyloadRouteHandler('./routes/weibo/keyword'));
-router.get('/weibo/search/hot', lazyloadRouteHandler('./routes/weibo/search/hot'));
-router.get('/weibo/super_index/:id/:type?/:routeParams?', lazyloadRouteHandler('./routes/weibo/super_index'));
-router.get('/weibo/oasis/user/:userid', lazyloadRouteHandler('./routes/weibo/oasis/user'));
+// 微博 migrated to v2
+// router.get('/weibo/user/:uid/:routeParams?', lazyloadRouteHandler('./routes/weibo/user'));
+// router.get('/weibo/keyword/:keyword/:routeParams?', lazyloadRouteHandler('./routes/weibo/keyword'));
+// router.get('/weibo/search/hot', lazyloadRouteHandler('./routes/weibo/search/hot'));
+// router.get('/weibo/super_index/:id/:type?/:routeParams?', lazyloadRouteHandler('./routes/weibo/super_index'));
+// router.get('/weibo/oasis/user/:userid', lazyloadRouteHandler('./routes/weibo/oasis/user'));
 
 // 贴吧 migrated to v2
 // router.get('/tieba/forum/:kw', lazyloadRouteHandler('./routes/tieba/forum'));
@@ -194,8 +194,8 @@ router.get('/dribbble/keyword/:keyword', lazyloadRouteHandler('./routes/dribbble
 router.get('/huya/live/:id', lazyloadRouteHandler('./routes/huya/live'));
 
 // 浪Play(原kingkong)直播
-router.get('/kingkong/room/:id', lazyloadRouteHandler('./routes/langlive/room'));
-router.get('/langlive/room/:id', lazyloadRouteHandler('./routes/langlive/room'));
+// router.get('/kingkong/room/:id', lazyloadRouteHandler('./routes/langlive/room'));
+// router.get('/langlive/room/:id', lazyloadRouteHandler('./routes/langlive/room'));
 
 // SHOWROOM直播
 router.get('/showroom/room/:id', lazyloadRouteHandler('./routes/showroom/room'));
@@ -569,9 +569,9 @@ router.get('/thu/:type', lazyloadRouteHandler('./routes/universities/thu/index')
 // router.get('/pku/bbs/hot', lazyloadRouteHandler('./routes/universities/pku/bbs/hot'));
 // router.get('/pku/scc/recruit/:type?', lazyloadRouteHandler('./routes/universities/pku/scc/recruit'));
 
-// 上海海事大学
-router.get('/shmtu/www/:type', lazyloadRouteHandler('./routes/universities/shmtu/www'));
-router.get('/shmtu/jwc/:type', lazyloadRouteHandler('./routes/universities/shmtu/jwc'));
+// 上海海事大学 migrated to v2
+// router.get('/shmtu/www/:type', lazyloadRouteHandler('./routes/universities/shmtu/www'));
+// router.get('/shmtu/jwc/:type', lazyloadRouteHandler('./routes/universities/shmtu/jwc'));
 
 // 上海海洋大学
 router.get('/shou/www/:type', lazyloadRouteHandler('./routes/universities/shou/www'));
@@ -1065,7 +1065,7 @@ router.get('/miniflux/:feeds/:parameters?', lazyloadRouteHandler('./routes/minif
 // router.get('/nga/post/:tid', lazyloadRouteHandler('./routes/nga/post'));
 
 // Nautilus
-router.get('/nautilus/topic/:tid', lazyloadRouteHandler('./routes/nautilus/topics'));
+// router.get('/nautilus/topic/:tid', lazyloadRouteHandler('./routes/nautilus/topics'));
 
 // JavBus migrated to v2
 // router.get('/javbus/home', lazyloadRouteHandler('./routes/javbus/home'));
@@ -1174,9 +1174,9 @@ router.get('/cyzone/author/:id', lazyloadRouteHandler('./routes/cyzone/author'))
 router.get('/cyzone/label/:name', lazyloadRouteHandler('./routes/cyzone/label'));
 
 // 政府
-router.get('/gov/zhengce/zuixin', lazyloadRouteHandler('./routes/gov/zhengce/zuixin'));
-router.get('/gov/zhengce/wenjian/:pcodeJiguan?', lazyloadRouteHandler('./routes/gov/zhengce/wenjian'));
-router.get('/gov/zhengce/govall/:advance?', lazyloadRouteHandler('./routes/gov/zhengce/govall'));
+// router.get('/gov/zhengce/zuixin', lazyloadRouteHandler('./routes/gov/zhengce/zuixin'));
+// router.get('/gov/zhengce/wenjian/:pcodeJiguan?', lazyloadRouteHandler('./routes/gov/zhengce/wenjian'));
+// router.get('/gov/zhengce/govall/:advance?', lazyloadRouteHandler('./routes/gov/zhengce/govall'));
 router.get('/gov/province/:name/:category', lazyloadRouteHandler('./routes/gov/province'));
 router.get('/gov/city/:name/:category', lazyloadRouteHandler('./routes/gov/city'));
 router.get('/gov/statecouncil/briefing', lazyloadRouteHandler('./routes/gov/statecouncil/briefing'));
@@ -1533,15 +1533,15 @@ router.get('/ps/:lang?/product/:gridName', lazyloadRouteHandler('./routes/ps/pro
 // Quanta Magazine
 router.get('/quantamagazine/archive', lazyloadRouteHandler('./routes/quantamagazine/archive'));
 
-// Nintendo
-router.get('/nintendo/eshop/jp', lazyloadRouteHandler('./routes/nintendo/eshop_jp'));
-router.get('/nintendo/eshop/hk', lazyloadRouteHandler('./routes/nintendo/eshop_hk'));
-router.get('/nintendo/eshop/us', lazyloadRouteHandler('./routes/nintendo/eshop_us'));
-router.get('/nintendo/eshop/cn', lazyloadRouteHandler('./routes/nintendo/eshop_cn'));
-router.get('/nintendo/news', lazyloadRouteHandler('./routes/nintendo/news'));
-router.get('/nintendo/news/china', lazyloadRouteHandler('./routes/nintendo/news_china'));
-router.get('/nintendo/direct', lazyloadRouteHandler('./routes/nintendo/direct'));
-router.get('/nintendo/system-update', lazyloadRouteHandler('./routes/nintendo/system-update'));
+// Nintendo migrated to v2
+// router.get('/nintendo/eshop/jp', lazyloadRouteHandler('./routes/nintendo/eshop_jp'));
+// router.get('/nintendo/eshop/hk', lazyloadRouteHandler('./routes/nintendo/eshop_hk'));
+// router.get('/nintendo/eshop/us', lazyloadRouteHandler('./routes/nintendo/eshop_us'));
+// router.get('/nintendo/eshop/cn', lazyloadRouteHandler('./routes/nintendo/eshop_cn'));
+// router.get('/nintendo/news', lazyloadRouteHandler('./routes/nintendo/news'));
+// router.get('/nintendo/news/china', lazyloadRouteHandler('./routes/nintendo/news_china'));
+// router.get('/nintendo/direct', lazyloadRouteHandler('./routes/nintendo/direct'));
+// router.get('/nintendo/system-update', lazyloadRouteHandler('./routes/nintendo/system-update'));
 
 // 世界卫生组织 migrated to v2
 // router.get('/who/news-room/:category?/:language?', lazyloadRouteHandler('./routes/who/news-room'));
@@ -2086,7 +2086,7 @@ router.get('/haimaoba/:id?', lazyloadRouteHandler('./routes/haimaoba/comics'));
 router.get('/pgyer/:app?', lazyloadRouteHandler('./routes/pgyer/app'));
 
 // 微博个人时间线
-router.get('/weibo/timeline/:uid/:feature?/:routeParams?', lazyloadRouteHandler('./routes/weibo/timeline'));
+// router.get('/weibo/timeline/:uid/:feature?/:routeParams?', lazyloadRouteHandler('./routes/weibo/timeline'));
 
 // TAPTAP migrated to v2
 // router.get('/taptap/topic/:id/:label?', lazyloadRouteHandler('./routes/taptap/topic'));
@@ -2578,8 +2578,8 @@ router.get('/zkyyjs', lazyloadRouteHandler('./routes/universities/zkyyjs/zkyyjs'
 // 中国海洋大学信电学院
 router.get('/outele', lazyloadRouteHandler('./routes/universities/outele/outele'));
 
-// 华东师范大学研究生院
-router.get('/ecnuyjs', lazyloadRouteHandler('./routes/universities/ecnuyjs/ecnuyjs'));
+// 华东师范大学研究生院 migrated to v2
+// router.get('/ecnuyjs', lazyloadRouteHandler('./routes/universities/ecnuyjs/ecnuyjs'));
 
 // 考研帮调剂信息
 router.get('/kaoyan', lazyloadRouteHandler('./routes/kaoyan/kaoyan'));
@@ -4028,7 +4028,7 @@ router.get('/sbs/chinese/:category?/:id?/:dialect?/:language?', lazyloadRouteHan
 // router.get('/asiantolick/:category?/:keyword?', lazyloadRouteHandler('./routes/asiantolick'));
 
 // Research Gate
-router.get('/researchgate/publications/:id', lazyloadRouteHandler('./routes/researchgate/publications'));
+// router.get('/researchgate/publications/:id', lazyloadRouteHandler('./routes/researchgate/publications'));
 
 // QuestMobile
 router.get('/questmobile/report/:category?/:label?', lazyloadRouteHandler('./routes/questmobile/report'));

package/lib/routes/v2ex/post.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const postid = ctx.params.postid;
@@ -29,8 +30,10 @@ module.exports = async (ctx) => {
                     guid: post.attr('id'),
                     link: `${pageUrl}#${post.attr('id')}`,
                     author: post.find('.dark').first().text(),
+                    pubDate: parseDate(post.find('.ago').attr('title')),
                 };
             })
             .reverse(),
+        allowEmpty: true,
     };
 };

package/lib/utils/puppeteer-utils.js

@@ -0,0 +1,63 @@
+/**
+ * Get Cookie-header-style cookie string from a puppeteer-style cookie array
+ *
+ * @param {import('puppeteer').Protocol.Network.CookieParam[]} cookies Puppeteer-style cookie array
+ * @param {RegExp | string} domainFilter Filter cookies by domain or RegExp
+ * @return {string} Cookie-header-style cookie string (e.g. "foobar; foo=bar; baz=qux")
+ */
+const parseCookieArray = (cookies, domainFilter = null) => {
+    if (typeof domainFilter === 'string') {
+        const dotDomain = '.' + domainFilter;
+        cookies = cookies.filter(({ domain }) => domain === domainFilter || domain.endsWith(dotDomain));
+    } else if (domainFilter && domainFilter.test !== undefined) {
+        cookies = cookies.filter(({ domain }) => domainFilter.test(domain));
+    }
+    // {name: '', value: 'foobar'} => 'foobar' // https://stackoverflow.com/questions/42531198/cookie-without-a-name
+    // {name: 'foo', value: 'bar'} => 'foo=bar'
+    return cookies.map(({ name, value }) => (name ? `${name}=${value}` : value)).join('; ');
+};
+
+/**
+ * Construct a puppeteer-style cookie array from a Cookie-header-style cookie string
+ *
+ * @param {string} cookieStr Cookie-header-style cookie string (e.g. "foobar; foo=bar; baz=qux")
+ * @param {string} domain Domain to set for each cookie
+ * @return {import('puppeteer').Protocol.Network.CookieParam[]} Puppeteer-style cookie array
+ */
+const constructCookieArray = (cookieStr, domain) =>
+    cookieStr.split('; ').map((item) => {
+        const [name, value] = item.split('=');
+        return value === undefined ? { name: '', value: name, domain } : { name, value, domain };
+    });
+
+/**
+ * Set cookies for a page
+ *
+ * @param {import('puppeteer').Page} page Puppeteer Page object
+ * @param {string} cookieStr Cookie-header-style cookie string (e.g. "foobar; foo=bar; baz=qux")
+ * @param {string} domain Domain to set for each cookie
+ * @return {Promise<void>}
+ */
+const setCookies = async (page, cookieStr, domain) => {
+    const cookies = constructCookieArray(cookieStr, domain);
+    await page.setCookie(...cookies);
+};
+
+/**
+ * Get Cookie-header-style cookie string from a page
+ *
+ * @param {import('puppeteer').Page} page Puppeteer Page object
+ * @param {RegExp | string} domainFilter Filter cookies by domain or RegExp
+ * @return {Promise<string>} Cookie-header-style cookie string
+ */
+const getCookies = async (page, domainFilter = null) => {
+    const cookies = await page.cookies();
+    return parseCookieArray(cookies, domainFilter);
+};
+
+module.exports = {
+    parseCookieArray,
+    constructCookieArray,
+    setCookies,
+    getCookies,
+};

package/lib/utils/puppeteer.js

@@ -1,6 +1,8 @@
 const config = require('@/config').value;
+const { proxyUri, proxyUrlHandler } = require('./unify-proxy');
 let puppeteer = require('puppeteer');
 const proxyChain = require('proxy-chain');
+const logger = require('./logger');
 
 const options = {
     args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
@@ -8,13 +10,6 @@ const options = {
     ignoreHTTPSErrors: true,
 };
 
-let proxyUri;
-if (config.proxyUri && typeof config.proxyUri === 'string') {
-    proxyUri = config.proxyUri;
-} else if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
-    proxyUri = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-}
-
 /**
  * @param {Object} extraOptions
  * @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth
@@ -50,7 +45,17 @@ module.exports = async (extraOptions = {}) => {
     }
     let browser;
     if (proxyUri) {
-        options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxyUri)}`);
+        if (proxyUrlHandler.username || proxyUrlHandler.password) {
+            // only proxies with authentication need to be anonymized
+            if (proxyUrlHandler.protocol === 'http:') {
+                options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxyUri)}`);
+            } else {
+                logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy');
+            }
+        } else {
+            // Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes
+            options.args.push(`--proxy-server=${proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
+        }
     }
     if (config.puppeteerWSEndpoint) {
         browser = await puppeteer.connect({

package/lib/utils/request-wrapper.js

@@ -1,127 +1,65 @@
 const config = require('@/config').value;
+const { proxyUri, proxyObj, proxyUrlHandler } = require('./unify-proxy');
 const logger = require('./logger');
 const http = require('http');
 const https = require('https');
 
-let tunnel;
-let HttpsProxyAgent;
-let SocksProxyAgent;
-
 let agent = null;
-if (config.proxyUri && typeof config.proxyUri === 'string') {
-    let proxy = null;
-    if (config.proxyUri.startsWith('http')) {
-        HttpsProxyAgent = HttpsProxyAgent || require('https-proxy-agent');
-        proxy = new HttpsProxyAgent(config.proxyUri);
-    } else if (config.proxyUri.startsWith('socks')) {
-        SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
-        proxy = new SocksProxyAgent(config.proxyUri);
-    } else {
-        throw 'Unknown proxy-uri format';
-    }
-
-    agent = {
-        http: proxy,
-        https: proxy,
-    };
-} else if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
-    agent = {};
-    const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-
-    switch (config.proxy.protocol) {
-        case 'socks':
-            SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
-            agent.http = new SocksProxyAgent(proxyUrl);
-            agent.https = new SocksProxyAgent(proxyUrl);
-            break;
-        case 'http':
-            tunnel = tunnel || require('tunnel');
-            process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
-            agent.http = tunnel.httpOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
-        case 'https':
-            tunnel = tunnel || require('tunnel');
-            agent.http = tunnel.httpOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
+if (proxyUri) {
+    if (proxyUri.startsWith('http')) {
+        const HttpsProxyAgent = require('https-proxy-agent');
+        agent = new HttpsProxyAgent(proxyUri);
+    } else if (proxyUri.startsWith('socks')) {
+        const SocksProxyAgent = require('socks-proxy-agent').SocksProxyAgent;
+        agent = new SocksProxyAgent(proxyUri);
     }
 }
 
-const requestWrapper = (url, options) => {
-    // agent
-    if (agent && new RegExp(config.proxy.url_regex).test(url)) {
-        let agentResult;
-        try {
-            agentResult = agent[(options.protocol || url.match(/(^https?:)/)[1]).slice(0, -1)];
-        } catch (error) {
-            agentResult = null;
-        }
-        try {
-            if (new URL(url).host !== new URL(config.proxyUri).host) {
-                options.agent = agentResult;
-            }
-        } catch (error) {
-            options.agent = agentResult;
-        }
+let proxyWrapper = () => false;
+if (agent) {
+    const proxyRegex = new RegExp(proxyObj.url_regex);
+    const protocolMatch = (protocolLike) => protocolLike && protocolLike.toLowerCase().startsWith('http');
 
-        if (config.proxy.auth) {
-            if (!options.headers) {
-                options.headers = {};
+    proxyWrapper = (url, options, urlHandler) => {
+        if (proxyRegex.test(url)) {
+            if ((protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler.host)) {
+                options.agent = agent;
+                if (proxyObj.auth) {
+                    options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
+                }
+                return true;
             }
-            options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
         }
-        logger.info(`Proxy for ${url}`);
-    } else {
-        logger.debug(`Requesting ${url}`);
+        return false;
+    };
+}
+
+const requestWrapper = (url, options) => {
+    let urlHandler;
+    try {
+        urlHandler = new URL(url);
+    } catch (error) {
+        // ignore
     }
+    options.headers = options.headers || {};
+    const headersLowerCaseKeys = Object.keys(options.headers).map((key) => key.toLowerCase());
+
+    proxyWrapper(url, options, urlHandler) ? logger.info(`Proxy for ${url}`) : logger.debug(`Requesting ${url}`);
 
     // ua
-    let hasUA = false;
-    for (const header in options.headers) {
-        if (header.toLowerCase() === 'user-agent') {
-            hasUA = true;
-        }
-    }
-    if (!hasUA) {
-        if (!options.headers) {
-            options.headers = {};
-        }
+    if (!headersLowerCaseKeys.includes('user-agent')) {
         options.headers['user-agent'] = config.ua;
     }
 
-    try {
-        const urlHandler = new URL(url);
+    if (urlHandler) {
         // referer
-        if (!options.headers.referer && !options.headers.Referer) {
+        if (!headersLowerCaseKeys.includes('referer')) {
             options.headers.referer = urlHandler.origin;
         }
         // host
-        if (!options.headers.host && !options.headers.Host) {
+        if (!headersLowerCaseKeys.includes('host')) {
             options.headers.host = urlHandler.host;
         }
-    } catch (e) {
-        // do nothing
     }
 };
 

package/lib/utils/unify-proxy.js

@@ -0,0 +1,116 @@
+const config = require('@/config').value;
+const logger = require('./logger');
+
+const defaultProtocol = 'http';
+const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h'];
+
+const unifyProxy = (proxyUri, proxyObj) => {
+    proxyObj = proxyObj || {};
+    const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj];
+    proxyObj = { ...proxyObj };
+
+    let proxyUrlHandler;
+
+    // PROXY_URI
+    if (proxyUri && typeof proxyUri === 'string') {
+        if (!proxyUri.includes('://')) {
+            logger.warn(`PROXY_URI contains no protocol, assuming ${defaultProtocol}`);
+            proxyUri = `${defaultProtocol}://${proxyUri}`;
+        }
+        try {
+            proxyUrlHandler = new URL(proxyUri);
+        } catch (e) {
+            logger.error(`Parse PROXY_URI error: ${e.stack}`);
+        }
+    }
+
+    // PROXY_{PROTOCOL,HOST,PORT}
+    if (proxyObj.protocol || proxyObj.host || proxyObj.port) {
+        if (proxyUrlHandler) {
+            logger.warn('PROXY_URI is set, ignoring PROXY_{PROTOCOL,HOST,PORT}');
+        } else if (proxyObj.host) {
+            let tempProxyStr = proxyObj.host;
+
+            if (tempProxyStr.includes('://')) {
+                logger.warn('PROXY_HOST contains protocol, ignoring PROXY_PROTOCOL');
+            } else if (!proxyObj.protocol) {
+                logger.warn(`PROXY_PROTOCOL is not set, assuming '${defaultProtocol}'`);
+                tempProxyStr = `${defaultProtocol}://${tempProxyStr}`;
+            } else {
+                tempProxyStr = `${proxyObj.protocol}://${tempProxyStr}`;
+            }
+            try {
+                proxyUrlHandler = new URL(tempProxyStr);
+                if (proxyUrlHandler.port && proxyObj.port) {
+                    logger.warn('PROXY_HOST contains port, ignoring PROXY_PORT');
+                } else if (proxyObj.port) {
+                    if (!parseInt(proxyObj.port)) {
+                        logger.warn(`PROXY_PORT is not a number, ignoring`);
+                    } else {
+                        proxyUrlHandler.port = proxyObj.port;
+                    }
+                } else {
+                    logger.warn('PROXY_PORT is not set, leaving proxy agent to determine');
+                }
+            } catch (e) {
+                logger.error(`Parse PROXY_HOST error: ${e.stack}`);
+            }
+        } else {
+            logger.warn('Either PROXY_{PROTOCOL,PORT} is set, but PROXY_HOST is missing, ignoring');
+        }
+    }
+
+    // PROXY_AUTH
+    if (proxyObj.auth && proxyUrlHandler) {
+        let promptProxyUri = false;
+        if (proxyUrlHandler.username || proxyUrlHandler.password) {
+            logger.warn('PROXY_URI contains username and/or password, ignoring PROXY_AUTH');
+            proxyObj.auth = undefined;
+        } else if (!['http:', 'https:'].includes(proxyUrlHandler.protocol)) {
+            logger.warn(`PROXY_AUTH is only supported by HTTP(S) proxies, but got ${proxyUrlHandler.protocol}, ignoring`);
+            proxyObj.auth = undefined;
+            promptProxyUri = true;
+        } else {
+            logger.info('PROXY_AUTH is set and will be used for requests from Node.js. However, requests from puppeteer will not use it');
+            promptProxyUri = true;
+        }
+        if (promptProxyUri) {
+            logger.info('To get rid of this, set PROXY_URI like protocol://username:password@host:port and clear PROXY_{AUTH,PROTOCOL,HOST,PORT}');
+        }
+    }
+
+    // is proxy enabled and valid?
+    let isProxyValid = false;
+    if (proxyUrlHandler) {
+        const protocol = proxyUrlHandler.protocol.replace(':', '');
+        if (possibleProtocol.includes(protocol)) {
+            if (protocol !== 'http' && (proxyUrlHandler.username || proxyUrlHandler.password)) {
+                logger.warn("PROXY_URI is an HTTPS/SOCKS proxy with authentication, which is not supported by puppeteer (ignore if you don't need it)");
+                logger.info('To get rid of this, consider using an HTTP proxy instead');
+            }
+            proxyObj.protocol = protocol;
+            proxyObj.host = proxyUrlHandler.hostname;
+            proxyObj.port = parseInt(proxyUrlHandler.port) || undefined;
+            // trailing slash will cause puppeteer to throw net::ERR_NO_SUPPORTED_PROXIES, trim it
+            proxyUri = proxyUrlHandler.href.endsWith('/') ? proxyUrlHandler.href.slice(0, -1) : proxyUrlHandler.href;
+            isProxyValid = true;
+        } else {
+            logger.error(`Unsupported proxy protocol: ${protocol}, expect one of ${possibleProtocol.join(', ')}`);
+        }
+    }
+    if (!isProxyValid) {
+        if ((oriProxyUri && typeof oriProxyUri === 'string') || oriProxyObj.protocol || oriProxyObj.host || oriProxyObj.port || oriProxyObj.auth) {
+            logger.error('Proxy is disabled due to misconfiguration');
+        }
+        proxyObj.protocol = proxyObj.host = proxyObj.port = proxyObj.auth = undefined;
+        proxyUri = null;
+        proxyUrlHandler = null;
+    }
+
+    return { proxyUri, proxyObj, proxyUrlHandler };
+};
+
+module.exports = {
+    unifyProxy,
+    ...unifyProxy(config.proxyUri, config.proxy),
+};

package/lib/utils/wechat-mp.js

@@ -8,6 +8,8 @@
  * lib/v2/pku/nsd/gd
  * lib/v2/sdu/cs
  * lib/v2/nua/utils
+ * lib/v2/hrbeu
+ * lib/v2/freewechat
  *
  * If your new route is not in the above folders, please add it to the list.
  *