rsshub 1.0.0-master.fe50f9c → 1.0.0-master.fe560d2

package/lib/utils/request-wrapper.js

@@ -1,127 +1,65 @@
 const config = require('@/config').value;
+const { proxyUri, proxyObj, proxyUrlHandler } = require('./unify-proxy');
 const logger = require('./logger');
 const http = require('http');
 const https = require('https');
 
-let tunnel;
-let HttpsProxyAgent;
-let SocksProxyAgent;
-
 let agent = null;
-if (config.proxyUri && typeof config.proxyUri === 'string') {
-    let proxy = null;
-    if (config.proxyUri.startsWith('http')) {
-        HttpsProxyAgent = HttpsProxyAgent || require('https-proxy-agent');
-        proxy = new HttpsProxyAgent(config.proxyUri);
-    } else if (config.proxyUri.startsWith('socks')) {
-        SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
-        proxy = new SocksProxyAgent(config.proxyUri);
-    } else {
-        throw 'Unknown proxy-uri format';
-    }
-
-    agent = {
-        http: proxy,
-        https: proxy,
-    };
-} else if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
-    agent = {};
-    const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-
-    switch (config.proxy.protocol) {
-        case 'socks':
-            SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
-            agent.http = new SocksProxyAgent(proxyUrl);
-            agent.https = new SocksProxyAgent(proxyUrl);
-            break;
-        case 'http':
-            tunnel = tunnel || require('tunnel');
-            process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
-            agent.http = tunnel.httpOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
-        case 'https':
-            tunnel = tunnel || require('tunnel');
-            agent.http = tunnel.httpOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
+if (proxyUri) {
+    if (proxyUri.startsWith('http')) {
+        const HttpsProxyAgent = require('https-proxy-agent');
+        agent = new HttpsProxyAgent(proxyUri);
+    } else if (proxyUri.startsWith('socks')) {
+        const SocksProxyAgent = require('socks-proxy-agent').SocksProxyAgent;
+        agent = new SocksProxyAgent(proxyUri);
     }
 }
 
-const requestWrapper = (url, options) => {
-    // agent
-    if (agent && new RegExp(config.proxy.url_regex).test(url)) {
-        let agentResult;
-        try {
-            agentResult = agent[(options.protocol || url.match(/(^https?:)/)[1]).slice(0, -1)];
-        } catch (error) {
-            agentResult = null;
-        }
-        try {
-            if (new URL(url).host !== new URL(config.proxyUri).host) {
-                options.agent = agentResult;
-            }
-        } catch (error) {
-            options.agent = agentResult;
-        }
+let proxyWrapper = () => false;
+if (agent) {
+    const proxyRegex = new RegExp(proxyObj.url_regex);
+    const protocolMatch = (protocolLike) => protocolLike && protocolLike.toLowerCase().startsWith('http');
 
-        if (config.proxy.auth) {
-            if (!options.headers) {
-                options.headers = {};
+    proxyWrapper = (url, options, urlHandler) => {
+        if (proxyRegex.test(url)) {
+            if ((protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler.host)) {
+                options.agent = agent;
+                if (proxyObj.auth) {
+                    options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
+                }
+                return true;
             }
-            options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
         }
-        logger.info(`Proxy for ${url}`);
-    } else {
-        logger.debug(`Requesting ${url}`);
+        return false;
+    };
+}
+
+const requestWrapper = (url, options) => {
+    let urlHandler;
+    try {
+        urlHandler = new URL(url);
+    } catch (error) {
+        // ignore
     }
+    options.headers = options.headers || {};
+    const headersLowerCaseKeys = Object.keys(options.headers).map((key) => key.toLowerCase());
+
+    proxyWrapper(url, options, urlHandler) ? logger.info(`Proxy for ${url}`) : logger.debug(`Requesting ${url}`);
 
     // ua
-    let hasUA = false;
-    for (const header in options.headers) {
-        if (header.toLowerCase() === 'user-agent') {
-            hasUA = true;
-        }
-    }
-    if (!hasUA) {
-        if (!options.headers) {
-            options.headers = {};
-        }
+    if (!headersLowerCaseKeys.includes('user-agent')) {
         options.headers['user-agent'] = config.ua;
     }
 
-    try {
-        const urlHandler = new URL(url);
+    if (urlHandler) {
         // referer
-        if (!options.headers.referer && !options.headers.Referer) {
+        if (!headersLowerCaseKeys.includes('referer')) {
             options.headers.referer = urlHandler.origin;
         }
         // host
-        if (!options.headers.host && !options.headers.Host) {
+        if (!headersLowerCaseKeys.includes('host')) {
             options.headers.host = urlHandler.host;
         }
-    } catch (e) {
-        // do nothing
     }
 };
 

package/lib/utils/unify-proxy.js

@@ -0,0 +1,116 @@
+const config = require('@/config').value;
+const logger = require('./logger');
+
+const defaultProtocol = 'http';
+const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h'];
+
+const unifyProxy = (proxyUri, proxyObj) => {
+    proxyObj = proxyObj || {};
+    const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj];
+    proxyObj = { ...proxyObj };
+
+    let proxyUrlHandler;
+
+    // PROXY_URI
+    if (proxyUri && typeof proxyUri === 'string') {
+        if (!proxyUri.includes('://')) {
+            logger.warn(`PROXY_URI contains no protocol, assuming ${defaultProtocol}`);
+            proxyUri = `${defaultProtocol}://${proxyUri}`;
+        }
+        try {
+            proxyUrlHandler = new URL(proxyUri);
+        } catch (e) {
+            logger.error(`Parse PROXY_URI error: ${e.stack}`);
+        }
+    }
+
+    // PROXY_{PROTOCOL,HOST,PORT}
+    if (proxyObj.protocol || proxyObj.host || proxyObj.port) {
+        if (proxyUrlHandler) {
+            logger.warn('PROXY_URI is set, ignoring PROXY_{PROTOCOL,HOST,PORT}');
+        } else if (proxyObj.host) {
+            let tempProxyStr = proxyObj.host;
+
+            if (tempProxyStr.includes('://')) {
+                logger.warn('PROXY_HOST contains protocol, ignoring PROXY_PROTOCOL');
+            } else if (!proxyObj.protocol) {
+                logger.warn(`PROXY_PROTOCOL is not set, assuming '${defaultProtocol}'`);
+                tempProxyStr = `${defaultProtocol}://${tempProxyStr}`;
+            } else {
+                tempProxyStr = `${proxyObj.protocol}://${tempProxyStr}`;
+            }
+            try {
+                proxyUrlHandler = new URL(tempProxyStr);
+                if (proxyUrlHandler.port && proxyObj.port) {
+                    logger.warn('PROXY_HOST contains port, ignoring PROXY_PORT');
+                } else if (proxyObj.port) {
+                    if (!parseInt(proxyObj.port)) {
+                        logger.warn(`PROXY_PORT is not a number, ignoring`);
+                    } else {
+                        proxyUrlHandler.port = proxyObj.port;
+                    }
+                } else {
+                    logger.warn('PROXY_PORT is not set, leaving proxy agent to determine');
+                }
+            } catch (e) {
+                logger.error(`Parse PROXY_HOST error: ${e.stack}`);
+            }
+        } else {
+            logger.warn('Either PROXY_{PROTOCOL,PORT} is set, but PROXY_HOST is missing, ignoring');
+        }
+    }
+
+    // PROXY_AUTH
+    if (proxyObj.auth && proxyUrlHandler) {
+        let promptProxyUri = false;
+        if (proxyUrlHandler.username || proxyUrlHandler.password) {
+            logger.warn('PROXY_URI contains username and/or password, ignoring PROXY_AUTH');
+            proxyObj.auth = undefined;
+        } else if (!['http:', 'https:'].includes(proxyUrlHandler.protocol)) {
+            logger.warn(`PROXY_AUTH is only supported by HTTP(S) proxies, but got ${proxyUrlHandler.protocol}, ignoring`);
+            proxyObj.auth = undefined;
+            promptProxyUri = true;
+        } else {
+            logger.info('PROXY_AUTH is set and will be used for requests from Node.js. However, requests from puppeteer will not use it');
+            promptProxyUri = true;
+        }
+        if (promptProxyUri) {
+            logger.info('To get rid of this, set PROXY_URI like protocol://username:password@host:port and clear PROXY_{AUTH,PROTOCOL,HOST,PORT}');
+        }
+    }
+
+    // is proxy enabled and valid?
+    let isProxyValid = false;
+    if (proxyUrlHandler) {
+        const protocol = proxyUrlHandler.protocol.replace(':', '');
+        if (possibleProtocol.includes(protocol)) {
+            if (protocol !== 'http' && (proxyUrlHandler.username || proxyUrlHandler.password)) {
+                logger.warn("PROXY_URI is an HTTPS/SOCKS proxy with authentication, which is not supported by puppeteer (ignore if you don't need it)");
+                logger.info('To get rid of this, consider using an HTTP proxy instead');
+            }
+            proxyObj.protocol = protocol;
+            proxyObj.host = proxyUrlHandler.hostname;
+            proxyObj.port = parseInt(proxyUrlHandler.port) || undefined;
+            // trailing slash will cause puppeteer to throw net::ERR_NO_SUPPORTED_PROXIES, trim it
+            proxyUri = proxyUrlHandler.href.endsWith('/') ? proxyUrlHandler.href.slice(0, -1) : proxyUrlHandler.href;
+            isProxyValid = true;
+        } else {
+            logger.error(`Unsupported proxy protocol: ${protocol}, expect one of ${possibleProtocol.join(', ')}`);
+        }
+    }
+    if (!isProxyValid) {
+        if ((oriProxyUri && typeof oriProxyUri === 'string') || oriProxyObj.protocol || oriProxyObj.host || oriProxyObj.port || oriProxyObj.auth) {
+            logger.error('Proxy is disabled due to misconfiguration');
+        }
+        proxyObj.protocol = proxyObj.host = proxyObj.port = proxyObj.auth = undefined;
+        proxyUri = null;
+        proxyUrlHandler = null;
+    }
+
+    return { proxyUri, proxyObj, proxyUrlHandler };
+};
+
+module.exports = {
+    unifyProxy,
+    ...unifyProxy(config.proxyUri, config.proxy),
+};

package/lib/utils/wechat-mp.js

@@ -7,6 +7,9 @@
  * lib/v2/gzh360
  * lib/v2/pku/nsd/gd
  * lib/v2/sdu/cs
+ * lib/v2/nua/utils
+ * lib/v2/hrbeu
+ * lib/v2/freewechat
  *
  * If your new route is not in the above folders, please add it to the list.
  *

package/lib/v2/18comic/utils.js

@@ -24,7 +24,7 @@ module.exports = {
 
                 return {
                     title: item.text().trim(),
-                    link: `${rootUrl}${item.prev().prev().attr('href')}`,
+                    link: `${rootUrl}${item.prev().find('a').attr('href')}`,
                 };
             });
 
@@ -52,7 +52,7 @@ module.exports = {
                         .join(', ');
                     item.description = `<p>${content('#intro-block .p-t-5').text()}</p><img src="${content('.img_zoom_img img')
                         .toArray()
-                        .map((image) => content(image).attr('src'))
+                        .map((image) => content(image).attr('data-original'))
                         .join('"><img src="')}">`;
 
                     return item;

package/lib/v2/36kr/index.js

@@ -27,16 +27,19 @@ module.exports = async (ctx) => {
 
     const data = JSON.parse(response.data.match(/"itemList":(\[.*?\])/)[1]);
 
-    let items = data.slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 30).map((item) => {
-        item = item.templateMaterial ?? item;
-        return {
-            title: item.widgetTitle.replace(/<\/?em>/g, ''),
-            author: item.author,
-            pubDate: parseDate(item.publishTime),
-            link: `${rootUrl}/${path === '/newsflashes' ? 'newsflashes' : 'p'}/${item.itemId}`,
-            description: item.widgetContent ?? item.content,
-        };
-    });
+    let items = data
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 30)
+        .filter((item) => item.itemType !== 0)
+        .map((item) => {
+            item = item.templateMaterial ?? item;
+            return {
+                title: item.widgetTitle.replace(/<\/?em>/g, ''),
+                author: item.author,
+                pubDate: parseDate(item.publishTime),
+                link: `${rootUrl}/${path === '/newsflashes' ? 'newsflashes' : 'p'}/${item.itemId}`,
+                description: item.widgetContent ?? item.content,
+            };
+        });
 
     if (!/^\/(search|newsflashes)/.test(path)) {
         items = await Promise.all(
@@ -47,20 +50,22 @@ module.exports = async (ctx) => {
                         url: item.link,
                     });
 
-                    const key = CryptoJS.enc.Utf8.parse('efabccee-b754-4c');
-                    const cipherText = detailResponse.data.match(/{"state":"(.*)","isEncrypt":true}/)[1];
-
-                    const content = JSON.parse(
-                        CryptoJS.AES.decrypt(cipherText, key, {
-                            mode: CryptoJS.mode.ECB,
-                            padding: CryptoJS.pad.Pkcs7,
-                        })
-                            .toString(CryptoJS.enc.Utf8)
-                            .toString()
-                    ).articleDetail.articleDetailData.data;
-
-                    item.description = content.widgetContent;
-
+                    const cipherTextList = detailResponse.data.match(/{"state":"(.*)","isEncrypt":true}/) ?? [];
+                    if (cipherTextList.length !== 0) {
+                        const key = CryptoJS.enc.Utf8.parse('efabccee-b754-4c');
+                        const content = JSON.parse(
+                            CryptoJS.AES.decrypt(cipherTextList[1], key, {
+                                mode: CryptoJS.mode.ECB,
+                                padding: CryptoJS.pad.Pkcs7,
+                            })
+                                .toString(CryptoJS.enc.Utf8)
+                                .toString()
+                        ).articleDetail.articleDetailData.data;
+                        item.description = content.widgetContent;
+                    } else {
+                        const $ = cheerio.load(detailResponse.body);
+                        item.description = $('div.articleDetailContent').html();
+                    }
                     return item;
                 })
             )

package/lib/v2/8world/index.js

@@ -3,10 +3,10 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const category = ctx.params.category ?? 'realtime';
+    const path = ctx.path === '/' ? '/realtime' : ctx.path;
 
     const rootUrl = 'https://www.8world.com';
-    const currentUrl = `${rootUrl}/${category}`;
+    const currentUrl = `${rootUrl}${path}`;
 
     const response = await got({
         method: 'get',
@@ -15,19 +15,19 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('div[data-column="Two-Third"] .article-title .article-link')
-        .map((_, item) => {
+    let items = $('div[data-column="Two-Third"] .article-title .article-link')
+        .toArray()
+        .map((item) => {
             item = $(item);
 
             return {
                 title: item.text(),
                 link: `${rootUrl}${item.attr('href')}`,
             };
-        })
-        .get();
+        });
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',

package/lib/v2/8world/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
     '/:category?': ['nczitzk'],
+    '/topic/:id': ['nczitzk'],
 };

package/lib/v2/8world/radar.js

@@ -8,6 +8,12 @@ module.exports = {
                 source: ['/:category', '/'],
                 target: '/8world/:category?',
             },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/new-media.html#_8-shi-jie-biao-qian',
+                source: ['/topic/:id', '/'],
+                target: '/8world/topic/:id',
+            },
         ],
     },
 };

package/lib/v2/8world/router.js

@@ -1,3 +1,3 @@
 module.exports = function (router) {
-    router.get('/:category?', require('./index'));
+    router.get(/([\w-/]+)?/, require('./index'));
 };

package/lib/v2/agora0/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
+    '/pen0': ['TonyRL'],
     '/:category?': ['nczitzk'],
 };

package/lib/v2/agora0/pen0.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'https://agorahub.github.io';
+    const response = await got(`${baseUrl}/pen0/`);
+    const $ = cheerio.load(response.data);
+
+    const list = $('div article')
+        .toArray()
+        .slice(0, -1) // last one is a dummy
+        .map((item) => {
+            item = $(item);
+            const meta = item.find('h5').first().text();
+            return {
+                title: item.find('h3').text(),
+                link: item.find('h3 a').attr('href'),
+                author: meta.split('|')[0].trim(),
+                pubDate: parseDate(meta.split('|')[1].trim()),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                $('h1').remove();
+                item.description = $('article').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('head meta[name="description"]').attr('content'),
+        link: response.url,
+        image: $('link[rel="apple-touch-icon"]').attr('href'),
+        item: items,
+    };
+};

package/lib/v2/agora0/radar.js

@@ -1,10 +1,21 @@
 module.exports = {
+    'github.io': {
+        _name: 'GitHub',
+        agorahub: [
+            {
+                title: '共和報',
+                docs: 'https://docs.rsshub.app/new-media.html#ag0ra',
+                source: ['/pen0'],
+                target: '/agora0/pen0',
+            },
+        ],
+    },
     'gitlab.io': {
         _name: 'GitLab',
         agora0: [
             {
                 title: '零博客',
-                docs: 'https://docs.rsshub.app/blog.html#ling-bo-ke-fen-lei',
+                docs: 'https://docs.rsshub.app/new-media.html#ag0ra',
                 source: ['/blog/:category', '/'],
                 target: '/agora0/:category',
             },

package/lib/v2/agora0/router.js

@@ -1,3 +1,4 @@
 module.exports = function (router) {
+    router.get('/pen0', require('./pen0'));
     router.get('/:category?', require('./index'));
 };

package/lib/v2/aiea/index.js

@@ -0,0 +1,39 @@
+const buildData = require('@/utils/common-config');
+
+module.exports = async (ctx) => {
+    const link = 'http://www.aiea.org/0504';
+    const period = ctx.params.period ?? '';
+
+    let nth_child = 'n';
+    switch (period) {
+        case 'upcoming':
+            nth_child = '1';
+            break;
+
+        case 'past':
+            nth_child = '2';
+            break;
+
+        case 'both':
+            nth_child = 'n';
+            break;
+
+        default:
+            break;
+    }
+
+    ctx.state.data = await buildData({
+        link,
+        url: link,
+        title: `%title%`,
+        params: {
+            title: 'AIEA Seminars',
+        },
+        item: {
+            item: `.seminar-contents .seminar-partWrap:nth-child(${nth_child}) > .seminar-list`,
+            title: `$('.seminar-list-title > span').text()`,
+            link: `$('a[href^="/0504"]').attr('href')`,
+            description: `$('.seminar-list .txt > .title').text()`,
+        },
+    });
+};

package/lib/v2/aiea/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/seminars/:period': ['zxx-457'],
+};

package/lib/v2/aiea/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'aiea.org': {
+        _name: 'Asian Innovation and Entrepreneurship Association',
+        www: [
+            {
+                title: 'Seminar Series',
+                docs: 'https://docs.rsshub.app/study.html#AIEA-Seminar-Series',
+                source: ['/0504', '/'],
+                target: '/aiea/seminars/upcoming',
+            },
+        ],
+    },
+};

package/lib/v2/aiea/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    return router.get('/seminars/:period', require('.'));
+};