rsshub 1.0.0-master.fb2a407 → 1.0.0-master.fc5a2f6

package/lib/utils/cf-email.js

@@ -0,0 +1,36 @@
+/**
+ * The alogrithm is well-explained in https://andrewlock.net/simple-obfuscation-of-email-addresses-using-javascript/
+ */
+module.exports = {
+    // eslint-disable-next-line lines-around-comment
+    /**
+     * Returns decoded email address using CloudFlare's email address obfuscation.
+     * @param {String} encoded - encoded email, (`cfemail` attribute in `__cf_email__` tag)
+     * @returns decoded email address
+     */
+    decodeCFEmail: (encoded) => {
+        const parseHex = (string, position) => parseInt(string.substr(position, 2), 16);
+        let decoded = '';
+        const key = parseHex(encoded, 0);
+        for (let position = 2; position < encoded.length; position += 2) {
+            const byte = parseHex(encoded, position) ^ key;
+            decoded += String.fromCharCode(byte);
+        }
+        return decoded;
+    },
+
+    /**
+     * Returns CloudFlare protected email address.
+     * @param {String} email - email in plaintext
+     * @returns obfuscated email
+     */
+    encodeCFEmail: (email) => {
+        const key = Math.floor(Math.random() * 256);
+        let encoded = ('0' + key.toString(16)).slice(-2);
+        for (let position = 0; position < email.length; position++) {
+            const byte = email.charCodeAt(position) ^ key;
+            encoded += ('0' + byte.toString(16)).slice(-2);
+        }
+        return encoded;
+    },
+};

package/lib/utils/parse-date.js

@@ -129,7 +129,7 @@ module.exports = {
 
         // 将 `\d+年\d+月...\d+秒前` 分割成 `['\d+年', ..., '\d+秒前']`
 
-        const matches = theDate.match(/(?:\D+)?\d+(?!:|-|\/)\D+/g);
+        const matches = theDate.match(/(?:\D+)?\d+(?!:|-|\/|(a|p)m)\D+/g);
 
         if (matches) {
             // 获得最后的时间单元，如 `\d+秒前`
@@ -187,7 +187,9 @@ module.exports = {
             for (const w of words) {
                 const wordMatches = w.regExp.exec(theDate);
                 if (wordMatches) {
-                    return dayjs(`${w.startAt.format('YYYY-MM-DD')} ${wordMatches[1]}`).toDate();
+                    // The default parser of dayjs() can parse '8:00 pm' but not '8:00pm'
+                    // so we need to insert a space in between
+                    return dayjs(`${w.startAt.format('YYYY-MM-DD')} ${/a|pm$/.test(wordMatches[1]) ? wordMatches[1].replace(/a|pm/, ' $&') : wordMatches[1]}`).toDate();
                 }
             }
         }

package/lib/utils/puppeteer.js

@@ -1,5 +1,5 @@
 const config = require('@/config').value;
-const puppeteer = require('puppeteer');
+let puppeteer = require('puppeteer');
 
 const options = {
     args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
@@ -7,14 +7,53 @@ const options = {
     ignoreHTTPSErrors: true,
 };
 
-module.exports = async () => {
+/**
+ * @param {Object} extraOptions
+ * @param {boolean} extraOptions.stealth - Use puppeteer-extra-plugin-stealth
+ * @returns Puppeteer browser
+ */
+module.exports = async (extraOptions = {}) => {
+    if (extraOptions.stealth) {
+        const { addExtra } = require('puppeteer-extra');
+        puppeteer = addExtra(puppeteer);
+
+        // workaround for vercel/nft #54 and #283
+        require('puppeteer-extra-plugin-stealth/evasions/chrome.app');
+        require('puppeteer-extra-plugin-stealth/evasions/chrome.csi');
+        require('puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes');
+        require('puppeteer-extra-plugin-stealth/evasions/chrome.runtime');
+        require('puppeteer-extra-plugin-stealth/evasions/defaultArgs');
+        require('puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow');
+        require('puppeteer-extra-plugin-stealth/evasions/media.codecs');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.languages');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.permissions');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.plugins');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.vendor');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.webdriver');
+        require('puppeteer-extra-plugin-stealth/evasions/sourceurl');
+        require('puppeteer-extra-plugin-stealth/evasions/user-agent-override');
+        require('puppeteer-extra-plugin-stealth/evasions/webgl.vendor');
+        require('puppeteer-extra-plugin-stealth/evasions/window.outerdimensions');
+        require('puppeteer-extra-plugin-user-preferences');
+        require('puppeteer-extra-plugin-user-data-dir');
+
+        puppeteer.use(require('puppeteer-extra-plugin-stealth')());
+    }
     let browser;
     if (config.puppeteerWSEndpoint) {
         browser = await puppeteer.connect({
             browserWSEndpoint: config.puppeteerWSEndpoint,
         });
     } else {
-        browser = await puppeteer.launch(options);
+        browser = await puppeteer.launch(
+            config.chromiumExecutablePath
+                ? {
+                      executablePath: config.chromiumExecutablePath,
+                      ...options,
+                  }
+                : options
+        );
     }
     setTimeout(() => {
         browser.close();

package/lib/utils/rand-user-agent.js

@@ -0,0 +1,29 @@
+const randUserAgent = require('rand-user-agent');
+
+/**
+ * A handy function to help generate a legit useragent.
+ *
+ * @param {Object} randUserAgent
+ * @param {string} randUserAgent.browser Name of a browser, case-insensitive. `chrome`, `edge`, `firefox`, `mobile safari`(ios only) or `safari`.
+ * @param {string} randUserAgent.os Name of an OS, case-insensitive. `android`, `ios`, `mac os`, `linux` or `windows`.
+ * @param {string} randUserAgent.device Name of a device, case-insensitive. `desktop`, `mobile` or `tablet`.
+ * @returns A random useragent for the given specifications.
+ */
+module.exports = ({ browser = 'chrome', os = 'mac os', device = 'desktop' }) => {
+    device = device.toLowerCase();
+    browser = browser.toLowerCase();
+    os = os.toLowerCase();
+    let UA = randUserAgent(device, browser, os);
+
+    if (browser === 'chrome') {
+        while (UA.includes('Chrome-Lighthouse') || UA.includes('Gener8') || UA.includes('HeadlessChrome') || UA.includes('SMTBot')) {
+            UA = randUserAgent(device, browser, os);
+        }
+    }
+    if (browser === 'safari') {
+        while (UA.includes('Applebot')) {
+            UA = randUserAgent(device, browser, os);
+        }
+    }
+    return UA;
+};

package/lib/utils/request-wrapper.js

@@ -14,7 +14,7 @@ if (config.proxyUri && typeof config.proxyUri === 'string') {
         HttpsProxyAgent = HttpsProxyAgent || require('https-proxy-agent');
         proxy = new HttpsProxyAgent(config.proxyUri);
     } else if (config.proxyUri.startsWith('socks')) {
-        SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent');
+        SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
         proxy = new SocksProxyAgent(config.proxyUri);
     } else {
         throw 'Unknown proxy-uri format';
@@ -30,7 +30,7 @@ if (config.proxyUri && typeof config.proxyUri === 'string') {
 
     switch (config.proxy.protocol) {
         case 'socks':
-            SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent');
+            SocksProxyAgent = SocksProxyAgent || require('socks-proxy-agent').SocksProxyAgent;
             agent.http = new SocksProxyAgent(proxyUrl);
             agent.https = new SocksProxyAgent(proxyUrl);
             break;
@@ -86,6 +86,8 @@ const requestWrapper = (url, options) => {
             options.headers['Proxy-Authorization'] = `Basic ${config.proxy.auth}`;
         }
         logger.info(`Proxy for ${url}`);
+    } else {
+        logger.debug(`Requesting ${url}`);
     }
 
     // ua
@@ -102,9 +104,6 @@ const requestWrapper = (url, options) => {
         options.headers['user-agent'] = config.ua;
     }
 
-    // server
-    options.headers.server = 'RSSHub';
-
     try {
         const urlHandler = new URL(url);
         // referer
@@ -128,7 +127,7 @@ const httpWrap = (func) => {
                 requestWrapper(url.toString(), request);
             } else {
                 const req = url;
-                requestWrapper(req.url || req.href || `${req.protocol}//${req.hostname}${req.path}`, req);
+                requestWrapper(req.url || req.href || `${req.protocol}//${req.hostname || req.host}${req.path}`, req);
             }
         } else {
             requestWrapper(url, request);

package/lib/utils/wechat-mp.js

@@ -2,9 +2,10 @@
  * Author: @Rongronggg9
  *
  * There are at least three folders which are relevant with WeChat MP (Official Account Platform / Media Platform):
- * lib/route/tencent/wechat
  * lib/v2/wechat
  * lib/v2/gzh360
+ * lib/v2/pku/nsd/gd
+ * lib/v2/sdu/cs
  *
  * If your new route is not in the above folders, please add it to the list.
  *
@@ -35,6 +36,31 @@ const replaceTag = ($, oldTag, newTagName) => {
     oldTag.replaceWith(NewTag);
 };
 
+const detectOriginalArticleUrl = ($) => {
+    // No article content get, try the original url
+    // example: https://mp.weixin.qq.com/s/f6sKObaZZhADTYU2Jl5Bnw
+    if (!$('#js_content').text()) {
+        return $('#js_share_source').attr('data-url');
+    }
+    // Article content is too short, try the first link
+    // example: https://mp.weixin.qq.com/s/9saVB4KaolRyJfpajzeFRg
+    if ($('#js_content').text().length < 80) {
+        return $('#js_content a').attr('href');
+    }
+    return null;
+};
+
+const detectSourceUrl = ($) => {
+    const matchs = $.root()
+        .html()
+        .match(/msg_source_url = '(.+)';/);
+
+    if (matchs) {
+        return matchs[1];
+    }
+    return null;
+};
+
 /**
  * Articles from WeChat MP have weird formats, this function is used to fix them.
  *
@@ -77,6 +103,18 @@ const fixArticleContent = (html, skipImg = false) => {
             replaceTag($, section, 'div');
         }
     });
+
+    // fix single picture article
+    // example: https://mp.weixin.qq.com/s/4p5YmYuASiQSYFiy7KqydQ
+    $('script').each((_, script) => {
+        script = $(script);
+        const matchs = script.html().match(/document\.getElementById\('js_image_desc'\)\.innerHTML = "(.*)"\.replace/);
+
+        if (matchs) {
+            script.replaceWith(matchs[1].replace(/\r/g, '').replace(/\n/g, '<br>').replace(/\\x0d/g, '').replace(/\\x0a/g, '<br>'));
+        }
+    });
+
     // clean scripts
     $('script').remove();
     return $.html();
@@ -143,11 +181,25 @@ const fetchArticle = async (ctx, url, bypassHostCheck = false) => {
         const response = await got(url);
         const $ = cheerio.load(response.data);
 
-        const title = $('meta[property="og:title"]').attr('content');
+        const title = ($('meta[property="og:title"]').attr('content') || '').replace(/\\r/g, '').replace(/\\n/g, ' ');
         const author = $('meta[name=author]').attr('content');
         let summary = $('meta[name=description]').attr('content');
         summary = summary !== title ? summary : '';
-        const description = fixArticleContent($('div#js_content.rich_media_content'));
+        let description = fixArticleContent($('#js_content'));
+
+        // No article get or article is too short, try the original url
+        const originalUrl = detectOriginalArticleUrl($);
+        if (originalUrl) {
+            // try to fetch the description from the original article
+            const originalResponse = await got(normalizeUrl(originalUrl, bypassHostCheck));
+            const original$ = cheerio.load(originalResponse.data);
+            description += fixArticleContent(original$('#js_content'));
+        }
+
+        const sourceUrl = detectSourceUrl($);
+        if (sourceUrl) {
+            description += `<a href="${sourceUrl}">阅读原文</a>`;
+        }
 
         let pubDate;
         const publish_time_script = $('script[nonce][type="text/javascript"]:contains("var ct")').first().html();

package/lib/v2/7mmtv/index.js

@@ -0,0 +1,86 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const language = ctx.params.language ?? 'en';
+    const category = ctx.params.category ?? 'censored_list';
+    const type = ctx.params.type ?? 'all';
+
+    const rootUrl = 'https://7mmtv.tv';
+    const currentUrl = `${rootUrl}/${language}/${category}/${type}/1.html`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    $('.pull-left p a').remove();
+
+    let items = $('.latest-korean-box-text a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const video = item.parent().parent().find('video');
+
+            return {
+                title: item.text(),
+                author: item.parent().find('.pull-left').text(),
+                pubDate: parseDate(item.parent().find('.date-part').text()),
+                link: `${item.attr('href').match(/^(http.*_content\/\d+)/)[1]}/index.html`,
+                poster: video.attr('poster'),
+                video: video.attr('srcmv'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    cover: content('.post-inner-details-img img').attr('src'),
+                    images: content('.video-introduction-images-list-row')
+                        .find('img')
+                        .toArray()
+                        .map((i) => content(i).attr('src')),
+                    poster: item.poster,
+                    video: item.video,
+                });
+
+                item.category = content('.actor-right-images-part p')
+                    .toArray()
+                    .map((a) => content(a).text())
+                    .concat(
+                        content('.posts-inner-details-text-under ul .posts-message span')
+                            .toArray()
+                            .map((c) => content(c).text())
+                    );
+
+                delete item.poster;
+                delete item.video;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title')
+            .text()
+            .replace(/ - Watch JAV Online/, ''),
+        link: currentUrl,
+        item: items,
+        description: $('meta[name="description"]').attr('content'),
+    };
+};

package/lib/v2/7mmtv/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/:language?/:category?/:type?': ['nczitzk'],
+    '/:language?/:category?/:id?': ['nczitzk'],
+};

package/lib/v2/7mmtv/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    '7mmtv.tv': {
+        _name: '7mmtv.tv',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/multimedia.html#7mmtv-fen-lei',
+                source: ['/'],
+                target: (params, url) => `/7mmtv/${new URL(url).toString().match(/\/(en|ja|ko|zh)\/([\w\d-]+\/){0,2}/)[1]}`,
+            },
+            {
+                title: '制作商',
+                docs: 'https://docs.rsshub.app/multimedia.html#7mmtv-zhi-zuo-shang',
+                source: ['/'],
+                target: (params, url) => `/7mmtv/${new URL(url).toString().match(/\/(en|ja|ko|zh)\/([\w\d-]+\/){0,2}/)[1]}`,
+            },
+        ],
+    },
+};

package/lib/v2/7mmtv/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:language?/:category?/:type?', require('./index'));
+};

package/lib/v2/7mmtv/templates/description.art

@@ -0,0 +1,11 @@
+{{ if cover }}
+<img src="{{ cover }}">
+{{ /if }}
+{{ if video }}
+<br>
+<video mute loop="loop" autoplay="autoplay" poster="{{ poster }}"><source src="{{ video }}"></video>
+<br>
+{{ /if }}
+{{ each images }}
+<img src="{{ $value }}">
+{{ /each }}

package/lib/v2/91porn/index.js

@@ -0,0 +1,58 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { domain = '91porn.com' } = ctx.query;
+    const siteUrl = `https://${domain}/index.php`;
+    const { lang = 'en_US' } = ctx.params;
+
+    const response = await got.post(siteUrl, {
+        form: {
+            session_language: lang,
+        },
+        headers: {
+            referer: siteUrl,
+        },
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.row .well')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.video-title').text(),
+                link: item.find('a').attr('href'),
+                poster: item.find('.img-responsive').attr('src'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(`91porn:${lang}:${new URL(item.link).searchParams.get('viewkey')}`, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+
+                item.pubDate = parseDate($('.title-yakov').eq(0).text(), 'YYYY-MM-DD');
+                item.description = art(path.join(__dirname, 'templates/index.art'), {
+                    link: item.link,
+                    poster: item.poster,
+                });
+                item.author = $('.title-yakov a span').text();
+                delete item.poster;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${$('.login_register_header').text()} - 91porn`,
+        link: siteUrl,
+        item: items,
+    };
+};

package/lib/v2/91porn/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:lang?': ['TonyRL'],
+};

package/lib/v2/91porn/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    '91porn.com': {
+        _name: '91porn',
+        '.': [
+            {
+                title: '91porn',
+                docs: 'https://docs.rsshub.app/multimedia.html#_91porn',
+                source: ['/index.php'],
+                target: '/91porn',
+            },
+        ],
+    },
+};

package/lib/v2/91porn/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:lang?', require('./index'));
+};

package/lib/v2/91porn/templates/index.art

@@ -0,0 +1 @@
+<a href="{{ link }}"><img src="{{ poster }} "></a>

package/lib/v2/9to5/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/9to5/:subsite/:tag?': ['HenryQW'],
+};

package/lib/v2/9to5/radar.js

@@ -0,0 +1,35 @@
+module.exports = {
+    '9to5toys.com': {
+        _name: '9to5',
+        '.': [
+            {
+                title: 'Toys 分站',
+                docs: 'https://docs.rsshub.app/new-media.html#_9to5',
+                source: ['/', '/guides/:tag'],
+                target: '/9to5/toys/:tag?',
+            },
+        ],
+    },
+    '9to5mac.com': {
+        _name: '9to5',
+        '.': [
+            {
+                title: 'Mac 分站',
+                docs: 'https://docs.rsshub.app/new-media.html#_9to5',
+                source: ['/', '/guides/:tag'],
+                target: '/9to5/mac/:tag?',
+            },
+        ],
+    },
+    '9to5google.com': {
+        _name: '9to5',
+        '.': [
+            {
+                title: 'Google 分站',
+                docs: 'https://docs.rsshub.app/new-media.html#_9to5',
+                source: ['/', '/guides/:tag'],
+                target: '/9to5/google/:tag?',
+            },
+        ],
+    },
+};

package/lib/v2/9to5/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:subsite/:tag?', require('./subsite'));
+};

package/lib/{routes → v2}/9to5/subsite.js

@@ -31,31 +31,34 @@ module.exports = async (ctx) => {
     }
 
     if (ctx.params.tag) {
-        link = `${link}/guides/${ctx.params.tag}/feed`;
+        link = `${link}/guides/${ctx.params.tag}/feed/`;
         title = `${ctx.params.tag} | ${title}`;
     } else {
-        link = `${link}/feed`;
+        link = `${link}/feed/`;
     }
 
     const feed = await parser.parseURL(link);
 
     const items = await Promise.all(
-        feed.items.splice(0, 10).map(async (item) => {
-            const response = await got({
-                method: 'get',
-                url: item.link,
-            });
-            const description = utils.ProcessFeed(response.data);
+        feed.items.splice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 10).map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const description = utils.ProcessFeed(response.data);
 
-            const single = {
-                title: item.title,
-                description,
-                pubDate: item.pubDate,
-                link: item.link,
-                author: item['dc:creator'],
-            };
-            return Promise.resolve(single);
-        })
+                const single = {
+                    title: item.title,
+                    description,
+                    pubDate: item.pubDate,
+                    link: item.link,
+                    author: item['dc:creator'],
+                };
+
+                return single;
+            })
+        )
     );
 
     ctx.state.data = {

package/lib/{routes → v2}/9to5/utils.js

@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
 
 const ProcessFeed = (data) => {
     const $ = cheerio.load(data);
-    const content = $('div[itemprop="articleBody"]');
+    const content = $('div.post-body');
 
     const cover = $('meta[property="og:image"]');
     if (cover.length > 0) {
@@ -16,6 +16,9 @@ const ProcessFeed = (data) => {
         $(e).remove();
     });
 
+    // remove ad
+    content.find('div.ad-disclaimer-container').remove();
+
     content.find('div').each((i, e) => {
         if ($(e)[0].attribs.class) {
             const classes = $(e)[0].attribs.class;