rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/routes/bbc/index.js

@@ -85,6 +85,7 @@ module.exports = async (ctx) => {
     ctx.state.data = {
         title,
         link,
+        image: 'https://www.bbc.com/favicon.ico',
         description: title,
         item: items,
     };

package/lib/routes/cell/cover.js

@@ -87,7 +87,7 @@ module.exports = async (ctx) => {
 
             const single = {
                 title: `${pageMeta.issueName} | ${pageMeta.issueNum} (${pageMeta.issueDate})`,
-                author: '@yech1990',
+                author: '@y9c',
                 description: contents,
                 link: address,
                 guid: address,

package/lib/routes/sankakucomplex/post.js

@@ -7,10 +7,7 @@ module.exports = async (ctx) => {
         url: 'https://idol.sankakucomplex.com/post/atom',
     });
 
-    const $ = cheerio.load(response.data, {
-        normalizeWhitespace: true,
-        xmlMode: true,
-    });
+    const $ = cheerio.load(response.data);
 
     ctx.state.data = {
         title: 'Posts | Idol Complex - Idol & Cosplay Images',

package/lib/routes/v2ex/post.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const postid = ctx.params.postid;
@@ -29,8 +30,10 @@ module.exports = async (ctx) => {
                     guid: post.attr('id'),
                     link: `${pageUrl}#${post.attr('id')}`,
                     author: post.find('.dark').first().text(),
+                    pubDate: parseDate(post.find('.ago').attr('title')),
                 };
             })
             .reverse(),
+        allowEmpty: true,
     };
 };

package/lib/routes/v2ex/topics.js

@@ -1,14 +1,10 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const type = ctx.params.type;
 
-    const response = await got({
-        method: 'get',
-        url: `https://www.v2ex.com/api/topics/${type}.json`,
-    });
-
-    const data = response.data;
+    const { data } = await got(`https://www.v2ex.com/api/topics/${type}.json`);
 
     let title;
     if (type === 'hot') {
@@ -25,10 +21,10 @@ module.exports = async (ctx) => {
             title: item.title,
             description: `${item.member.username}: ${item.content_rendered}`,
             content: { text: item.content, html: item.content_rendered },
-            pubDate: new Date(item.created * 1000).toUTCString(),
-            guid: item.id,
+            pubDate: parseDate(item.created, 'X'),
             link: item.url,
             author: item.member.username,
+            comments: item.replies,
         })),
     };
 };

package/lib/utils/parse-date.js

@@ -70,19 +70,19 @@ const patterns = [
     },
     {
         unit: 'days',
-        regExp: /(\d+)(?:天|日|day(?:s)?)/,
+        regExp: /(\d+)(?:天|日|d(?:ay)?(?:s)?)/,
     },
     {
         unit: 'hours',
-        regExp: /(\d+)(?:(?:个|個)?(?:(?:小)?时|時|点|點)|h(?:ou)?r(?:s)?)/,
+        regExp: /(\d+)(?:(?:个|個)?(?:(?:小)?时|時|点|點)|h(?:(?:ou)?r)?(?:s)?)/,
     },
     {
         unit: 'minutes',
-        regExp: /(\d+)(?:分(?:钟|鐘)?|min(?:ute)?(?:s)?)/,
+        regExp: /(\d+)(?:分(?:钟|鐘)?|m(?:in(?:ute)?)?(?:s)?)/,
     },
     {
         unit: 'seconds',
-        regExp: /(\d+)(?:秒(?:钟|鐘)?|sec(?:ond)?(?:s)?)/,
+        regExp: /(\d+)(?:秒(?:钟|鐘)?|s(?:ec(?:ond)?)?(?:s)?)/,
     },
 ];
 

package/lib/utils/puppeteer-utils.js

@@ -0,0 +1,63 @@
+/**
+ * Get Cookie-header-style cookie string from a puppeteer-style cookie array
+ *
+ * @param {import('puppeteer').Protocol.Network.CookieParam[]} cookies Puppeteer-style cookie array
+ * @param {RegExp | string} domainFilter Filter cookies by domain or RegExp
+ * @return {string} Cookie-header-style cookie string (e.g. "foobar; foo=bar; baz=qux")
+ */
+const parseCookieArray = (cookies, domainFilter = null) => {
+    if (typeof domainFilter === 'string') {
+        const dotDomain = '.' + domainFilter;
+        cookies = cookies.filter(({ domain }) => domain === domainFilter || domain.endsWith(dotDomain));
+    } else if (domainFilter && domainFilter.test !== undefined) {
+        cookies = cookies.filter(({ domain }) => domainFilter.test(domain));
+    }
+    // {name: '', value: 'foobar'} => 'foobar' // https://stackoverflow.com/questions/42531198/cookie-without-a-name
+    // {name: 'foo', value: 'bar'} => 'foo=bar'
+    return cookies.map(({ name, value }) => (name ? `${name}=${value}` : value)).join('; ');
+};
+
+/**
+ * Construct a puppeteer-style cookie array from a Cookie-header-style cookie string
+ *
+ * @param {string} cookieStr Cookie-header-style cookie string (e.g. "foobar; foo=bar; baz=qux")
+ * @param {string} domain Domain to set for each cookie
+ * @return {import('puppeteer').Protocol.Network.CookieParam[]} Puppeteer-style cookie array
+ */
+const constructCookieArray = (cookieStr, domain) =>
+    cookieStr.split('; ').map((item) => {
+        const [name, value] = item.split('=');
+        return value === undefined ? { name: '', value: name, domain } : { name, value, domain };
+    });
+
+/**
+ * Set cookies for a page
+ *
+ * @param {import('puppeteer').Page} page Puppeteer Page object
+ * @param {string} cookieStr Cookie-header-style cookie string (e.g. "foobar; foo=bar; baz=qux")
+ * @param {string} domain Domain to set for each cookie
+ * @return {Promise<void>}
+ */
+const setCookies = async (page, cookieStr, domain) => {
+    const cookies = constructCookieArray(cookieStr, domain);
+    await page.setCookie(...cookies);
+};
+
+/**
+ * Get Cookie-header-style cookie string from a page
+ *
+ * @param {import('puppeteer').Page} page Puppeteer Page object
+ * @param {RegExp | string} domainFilter Filter cookies by domain or RegExp
+ * @return {Promise<string>} Cookie-header-style cookie string
+ */
+const getCookies = async (page, domainFilter = null) => {
+    const cookies = await page.cookies();
+    return parseCookieArray(cookies, domainFilter);
+};
+
+module.exports = {
+    parseCookieArray,
+    constructCookieArray,
+    setCookies,
+    getCookies,
+};

package/lib/utils/puppeteer.js

@@ -1,5 +1,8 @@
 const config = require('@/config').value;
+const { proxyUri, proxyUrlHandler } = require('./unify-proxy');
 let puppeteer = require('puppeteer');
+const proxyChain = require('proxy-chain');
+const logger = require('./logger');
 
 const options = {
     args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
@@ -41,6 +44,19 @@ module.exports = async (extraOptions = {}) => {
         puppeteer.use(require('puppeteer-extra-plugin-stealth')());
     }
     let browser;
+    if (proxyUri) {
+        if (proxyUrlHandler.username || proxyUrlHandler.password) {
+            // only proxies with authentication need to be anonymized
+            if (proxyUrlHandler.protocol === 'http:') {
+                options.args.push(`--proxy-server=${await proxyChain.anonymizeProxy(proxyUri)}`);
+            } else {
+                logger.warn('SOCKS/HTTPS proxy with authentication is not supported by puppeteer, continue without proxy');
+            }
+        } else {
+            // Chromium cannot recognize socks5h and socks4a, so we need to trim their postfixes
+            options.args.push(`--proxy-server=${proxyUri.replace('socks5h://', 'socks5://').replace('socks4a://', 'socks4://')}`);
+        }
+    }
     if (config.puppeteerWSEndpoint) {
         browser = await puppeteer.connect({
             browserWSEndpoint: config.puppeteerWSEndpoint,

package/lib/utils/render.js

@@ -2,6 +2,51 @@ const art = require('art-template');
 
 // We may add more control over it later
 
+/**
+ * This function should be used by RSSHub middleware only.
+ * @param {object} data ctx.state.data
+ * @returns `JSON.stringify`-ed [JSON Feed](https://www.jsonfeed.org/)
+ */
+const json = (data) => {
+    const jsonFeed = {
+        version: 'https://jsonfeed.org/version/1.1',
+        title: data.title || 'RSSHub',
+        home_page_url: data.link || 'https://docs.rsshub.app',
+        feed_url: data.feedLink,
+        description: `${data.description || data.title} - Made with love by RSSHub(https://github.com/DIYgod/RSSHub)`,
+        icon: data.image,
+        authors: typeof data.author === 'string' ? [{ name: data.author }] : data.author,
+        language: data.language || 'zh-cn',
+        items: data.item.map((item) => ({
+            id: item.guid || item.id || item.link,
+            url: item.link,
+            title: item.title,
+            content_html: (item.content && item.content.html) || item.description || item.title,
+            content_text: item.content && item.content.text,
+            image: item.image,
+            banner_image: item.banner,
+            date_published: item.pubDate,
+            date_modified: item.updated,
+            authors: typeof item.author === 'string' ? [{ name: item.author }] : item.author,
+            tags: typeof item.category === 'string' ? [item.category] : item.category,
+            language: item.language,
+            attachments: item.enclosure_url
+                ? [
+                      {
+                          url: item.enclosure_url,
+                          mime_type: item.enclosure_type,
+                          title: item.enclosure_title,
+                          size_in_bytes: item.enclosure_length,
+                          duration_in_seconds: item.itunes_duration,
+                      },
+                  ]
+                : undefined,
+        })),
+    };
+    return JSON.stringify(jsonFeed, null, 4);
+};
+
 module.exports = {
     art,
+    json,
 };

package/lib/utils/request-wrapper.js

@@ -1,121 +1,65 @@
 const config = require('@/config').value;
+const { proxyUri, proxyObj, proxyUrlHandler } = require('./unify-proxy');
 const logger = require('./logger');
 const http = require('http');
 const https = require('https');
 
-let tunnel;
-let HttpsProxyAgent;
-let SocksProxyAgent;
-
 let agent = null;
-if (config.proxyUri && typeof config.proxyUri === 'string') {
-    let proxy = null;
-    if (config.proxyUri.startsWith('http')) {
-        HttpsProxyAgent = HttpsProxyAgent || require('https-proxy-agent');
-        proxy = new HttpsProxyAgent(config.proxyUri);
-    } else if (config.proxyUri.startsWith('socks')) {
-        SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
-        proxy = new SocksProxyAgent(config.proxyUri);
-    } else {
-        throw 'Unknown proxy-uri format';
-    }
-
-    agent = {
-        http: proxy,
-        https: proxy,
-    };
-} else if (config.proxy && config.proxy.protocol && config.proxy.host && config.proxy.port) {
-    agent = {};
-    const proxyUrl = `${config.proxy.protocol}://${config.proxy.host}:${config.proxy.port}`;
-
-    switch (config.proxy.protocol) {
-        case 'socks':
-            SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
-            agent.http = new SocksProxyAgent(proxyUrl);
-            agent.https = new SocksProxyAgent(proxyUrl);
-            break;
-        case 'http':
-            tunnel = tunnel || require('tunnel');
-            process.env.NODE_TLS_REJECT_UNAUTHORIZED = 0;
-            agent.http = tunnel.httpOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttp({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
-        case 'https':
-            tunnel = tunnel || require('tunnel');
-            agent.http = tunnel.httpOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            agent.https = tunnel.httpsOverHttps({
-                proxy: {
-                    host: config.proxy.host,
-                    port: parseInt(config.proxy.port),
-                },
-            });
-            break;
+if (proxyUri) {
+    if (proxyUri.startsWith('http')) {
+        const HttpsProxyAgent = require('https-proxy-agent');
+        agent = new HttpsProxyAgent(proxyUri);
+    } else if (proxyUri.startsWith('socks')) {
+        const SocksProxyAgent = require('socks-proxy-agent').SocksProxyAgent;
+        agent = new SocksProxyAgent(proxyUri);
     }
 }
 
-const requestWrapper = (url, options) => {
-    // agent
-    if (agent && new RegExp(config.proxy.url_regex).test(url)) {
-        let agentResult;
-        try {
-            agentResult = agent[(options.protocol || url.match(/(https?:)/)[1]).slice(0, -1)];
-        } catch (error) {
-            agentResult = null;
-        }
-        options.agent = agentResult;
+let proxyWrapper = () => false;
+if (agent) {
+    const proxyRegex = new RegExp(proxyObj.url_regex);
+    const protocolMatch = (protocolLike) => protocolLike && protocolLike.toLowerCase().startsWith('http');
 
-        if (config.proxy.auth) {
-            if (!options.headers) {
-                options.headers = {};
+    proxyWrapper = (url, options, urlHandler) => {
+        if (proxyRegex.test(url)) {
+            if ((protocolMatch(options.protocol) || protocolMatch(url)) && (!urlHandler || urlHandler.host !== proxyUrlHandler.host)) {
+                options.agent = agent;
+                if (proxyObj.auth) {
+                    options.headers['Proxy-Authorization'] = `Basic ${proxyObj.auth}`;
+                }
+                return true;
             }
-            options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
         }
-        logger.info(`Proxy for ${url}`);
-    } else {
-        logger.debug(`Requesting ${url}`);
+        return false;
+    };
+}
+
+const requestWrapper = (url, options) => {
+    let urlHandler;
+    try {
+        urlHandler = new URL(url);
+    } catch (error) {
+        // ignore
     }
+    options.headers = options.headers || {};
+    const headersLowerCaseKeys = Object.keys(options.headers).map((key) => key.toLowerCase());
+
+    proxyWrapper(url, options, urlHandler) ? logger.info(`Proxy for ${url}`) : logger.debug(`Requesting ${url}`);
 
     // ua
-    let hasUA = false;
-    for (const header in options.headers) {
-        if (header.toLowerCase() === 'user-agent') {
-            hasUA = true;
-        }
-    }
-    if (!hasUA) {
-        if (!options.headers) {
-            options.headers = {};
-        }
+    if (!headersLowerCaseKeys.includes('user-agent')) {
         options.headers['user-agent'] = config.ua;
     }
 
-    try {
-        const urlHandler = new URL(url);
+    if (urlHandler) {
         // referer
-        if (!options.headers.referer && !options.headers.Referer) {
+        if (!headersLowerCaseKeys.includes('referer')) {
             options.headers.referer = urlHandler.origin;
         }
         // host
-        if (!options.headers.host && !options.headers.Host) {
+        if (!headersLowerCaseKeys.includes('host')) {
             options.headers.host = urlHandler.host;
         }
-    } catch (e) {
-        // do nothing
     }
 };
 

package/lib/utils/unify-proxy.js

@@ -0,0 +1,116 @@
+const config = require('@/config').value;
+const logger = require('./logger');
+
+const defaultProtocol = 'http';
+const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h'];
+
+const unifyProxy = (proxyUri, proxyObj) => {
+    proxyObj = proxyObj || {};
+    const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj];
+    proxyObj = { ...proxyObj };
+
+    let proxyUrlHandler;
+
+    // PROXY_URI
+    if (proxyUri && typeof proxyUri === 'string') {
+        if (!proxyUri.includes('://')) {
+            logger.warn(`PROXY_URI contains no protocol, assuming ${defaultProtocol}`);
+            proxyUri = `${defaultProtocol}://${proxyUri}`;
+        }
+        try {
+            proxyUrlHandler = new URL(proxyUri);
+        } catch (e) {
+            logger.error(`Parse PROXY_URI error: ${e.stack}`);
+        }
+    }
+
+    // PROXY_{PROTOCOL,HOST,PORT}
+    if (proxyObj.protocol || proxyObj.host || proxyObj.port) {
+        if (proxyUrlHandler) {
+            logger.warn('PROXY_URI is set, ignoring PROXY_{PROTOCOL,HOST,PORT}');
+        } else if (proxyObj.host) {
+            let tempProxyStr = proxyObj.host;
+
+            if (tempProxyStr.includes('://')) {
+                logger.warn('PROXY_HOST contains protocol, ignoring PROXY_PROTOCOL');
+            } else if (!proxyObj.protocol) {
+                logger.warn(`PROXY_PROTOCOL is not set, assuming '${defaultProtocol}'`);
+                tempProxyStr = `${defaultProtocol}://${tempProxyStr}`;
+            } else {
+                tempProxyStr = `${proxyObj.protocol}://${tempProxyStr}`;
+            }
+            try {
+                proxyUrlHandler = new URL(tempProxyStr);
+                if (proxyUrlHandler.port && proxyObj.port) {
+                    logger.warn('PROXY_HOST contains port, ignoring PROXY_PORT');
+                } else if (proxyObj.port) {
+                    if (!parseInt(proxyObj.port)) {
+                        logger.warn(`PROXY_PORT is not a number, ignoring`);
+                    } else {
+                        proxyUrlHandler.port = proxyObj.port;
+                    }
+                } else {
+                    logger.warn('PROXY_PORT is not set, leaving proxy agent to determine');
+                }
+            } catch (e) {
+                logger.error(`Parse PROXY_HOST error: ${e.stack}`);
+            }
+        } else {
+            logger.warn('Either PROXY_{PROTOCOL,PORT} is set, but PROXY_HOST is missing, ignoring');
+        }
+    }
+
+    // PROXY_AUTH
+    if (proxyObj.auth && proxyUrlHandler) {
+        let promptProxyUri = false;
+        if (proxyUrlHandler.username || proxyUrlHandler.password) {
+            logger.warn('PROXY_URI contains username and/or password, ignoring PROXY_AUTH');
+            proxyObj.auth = undefined;
+        } else if (!['http:', 'https:'].includes(proxyUrlHandler.protocol)) {
+            logger.warn(`PROXY_AUTH is only supported by HTTP(S) proxies, but got ${proxyUrlHandler.protocol}, ignoring`);
+            proxyObj.auth = undefined;
+            promptProxyUri = true;
+        } else {
+            logger.info('PROXY_AUTH is set and will be used for requests from Node.js. However, requests from puppeteer will not use it');
+            promptProxyUri = true;
+        }
+        if (promptProxyUri) {
+            logger.info('To get rid of this, set PROXY_URI like protocol://username:password@host:port and clear PROXY_{AUTH,PROTOCOL,HOST,PORT}');
+        }
+    }
+
+    // is proxy enabled and valid?
+    let isProxyValid = false;
+    if (proxyUrlHandler) {
+        const protocol = proxyUrlHandler.protocol.replace(':', '');
+        if (possibleProtocol.includes(protocol)) {
+            if (protocol !== 'http' && (proxyUrlHandler.username || proxyUrlHandler.password)) {
+                logger.warn("PROXY_URI is an HTTPS/SOCKS proxy with authentication, which is not supported by puppeteer (ignore if you don't need it)");
+                logger.info('To get rid of this, consider using an HTTP proxy instead');
+            }
+            proxyObj.protocol = protocol;
+            proxyObj.host = proxyUrlHandler.hostname;
+            proxyObj.port = parseInt(proxyUrlHandler.port) || undefined;
+            // trailing slash will cause puppeteer to throw net::ERR_NO_SUPPORTED_PROXIES, trim it
+            proxyUri = proxyUrlHandler.href.endsWith('/') ? proxyUrlHandler.href.slice(0, -1) : proxyUrlHandler.href;
+            isProxyValid = true;
+        } else {
+            logger.error(`Unsupported proxy protocol: ${protocol}, expect one of ${possibleProtocol.join(', ')}`);
+        }
+    }
+    if (!isProxyValid) {
+        if ((oriProxyUri && typeof oriProxyUri === 'string') || oriProxyObj.protocol || oriProxyObj.host || oriProxyObj.port || oriProxyObj.auth) {
+            logger.error('Proxy is disabled due to misconfiguration');
+        }
+        proxyObj.protocol = proxyObj.host = proxyObj.port = proxyObj.auth = undefined;
+        proxyUri = null;
+        proxyUrlHandler = null;
+    }
+
+    return { proxyUri, proxyObj, proxyUrlHandler };
+};
+
+module.exports = {
+    unifyProxy,
+    ...unifyProxy(config.proxyUri, config.proxy),
+};

package/lib/utils/wechat-mp.js

@@ -7,6 +7,9 @@
  * lib/v2/gzh360
  * lib/v2/pku/nsd/gd
  * lib/v2/sdu/cs
+ * lib/v2/nua/utils
+ * lib/v2/hrbeu
+ * lib/v2/freewechat
  *
  * If your new route is not in the above folders, please add it to the list.
  *

package/lib/v2/18comic/utils.js

@@ -24,7 +24,7 @@ module.exports = {
 
                 return {
                     title: item.text().trim(),
-                    link: `${rootUrl}${item.prev().prev().attr('href')}`,
+                    link: `${rootUrl}${item.prev().find('a').attr('href')}`,
                 };
             });
 
@@ -52,7 +52,7 @@ module.exports = {
                         .join(', ');
                     item.description = `<p>${content('#intro-block .p-t-5').text()}</p><img src="${content('.img_zoom_img img')
                         .toArray()
-                        .map((image) => content(image).attr('src'))
+                        .map((image) => content(image).attr('data-original'))
                         .join('"><img src="')}">`;
 
                     return item;

package/lib/v2/35photo/genre.js

@@ -4,6 +4,7 @@ module.exports = async (ctx) => {
     const id = ctx.params.id;
 
     const currentUrl = `${rootUrl}/genre_${id}`;
+    const apiUrl = `${rootUrl}/show_block.php?type=getNextPageData&page=genre&community_id=${id}`;
 
-    ctx.state.data = await ProcessItems(currentUrl);
+    ctx.state.data = await ProcessItems(currentUrl, apiUrl);
 };

package/lib/v2/35photo/utils.js

@@ -5,13 +5,28 @@ const rootUrl = 'https://35photo.pro';
 
 module.exports = {
     rootUrl,
-    ProcessItems: async (currentUrl) => {
-        const response = await got({
+    ProcessItems: async (currentUrl, apiUrl) => {
+        let $, response;
+
+        response = await got({
             method: 'get',
             url: currentUrl,
         });
 
-        const $ = cheerio.load(response.data);
+        const data = response.data;
+
+        $ = cheerio.load(data);
+
+        const title = $('h1').text();
+
+        if (apiUrl) {
+            response = await got({
+                method: 'get',
+                url: `${apiUrl}&lastId=${$('div.countLike').last().attr('photo-id')}`,
+            });
+
+            $ = cheerio.load(data + response.data.data);
+        }
 
         const items = $('.item')
             .toArray()
@@ -34,7 +49,7 @@ module.exports = {
             });
 
         return {
-            title: `${$('h1').text()} - 35PHOTO`,
+            title: `${title} - 35PHOTO`,
             link: currentUrl,
             item: items,
         };