rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/lib/utils/parse-date.js

@@ -129,7 +129,7 @@ module.exports = {
 
         // 将 `\d+年\d+月...\d+秒前` 分割成 `['\d+年', ..., '\d+秒前']`
 
-        const matches = theDate.match(/(?:\D+)?\d+(?!:|-|\/)\D+/g);
+        const matches = theDate.match(/(?:\D+)?\d+(?!:|-|\/|(a|p)m)\D+/g);
 
         if (matches) {
             // 获得最后的时间单元，如 `\d+秒前`
@@ -187,7 +187,9 @@ module.exports = {
             for (const w of words) {
                 const wordMatches = w.regExp.exec(theDate);
                 if (wordMatches) {
-                    return dayjs(`${w.startAt.format('YYYY-MM-DD')} ${wordMatches[1]}`).toDate();
+                    // The default parser of dayjs() can parse '8:00 pm' but not '8:00pm'
+                    // so we need to insert a space in between
+                    return dayjs(`${w.startAt.format('YYYY-MM-DD')} ${/a|pm$/.test(wordMatches[1]) ? wordMatches[1].replace(/a|pm/, ' $&') : wordMatches[1]}`).toDate();
                 }
             }
         }

package/lib/utils/puppeteer.js

@@ -16,6 +16,28 @@ module.exports = async (extraOptions = {}) => {
     if (extraOptions.stealth) {
         const { addExtra } = require('puppeteer-extra');
         puppeteer = addExtra(puppeteer);
+
+        // workaround for vercel/nft #54 and #283
+        require('puppeteer-extra-plugin-stealth/evasions/chrome.app');
+        require('puppeteer-extra-plugin-stealth/evasions/chrome.csi');
+        require('puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes');
+        require('puppeteer-extra-plugin-stealth/evasions/chrome.runtime');
+        require('puppeteer-extra-plugin-stealth/evasions/defaultArgs');
+        require('puppeteer-extra-plugin-stealth/evasions/iframe.contentWindow');
+        require('puppeteer-extra-plugin-stealth/evasions/media.codecs');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.languages');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.permissions');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.plugins');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.vendor');
+        require('puppeteer-extra-plugin-stealth/evasions/navigator.webdriver');
+        require('puppeteer-extra-plugin-stealth/evasions/sourceurl');
+        require('puppeteer-extra-plugin-stealth/evasions/user-agent-override');
+        require('puppeteer-extra-plugin-stealth/evasions/webgl.vendor');
+        require('puppeteer-extra-plugin-stealth/evasions/window.outerdimensions');
+        require('puppeteer-extra-plugin-user-preferences');
+        require('puppeteer-extra-plugin-user-data-dir');
+
         puppeteer.use(require('puppeteer-extra-plugin-stealth')());
     }
     let browser;

package/lib/utils/wechat-mp.js

@@ -4,6 +4,7 @@
  * There are at least three folders which are relevant with WeChat MP (Official Account Platform / Media Platform):
  * lib/v2/wechat
  * lib/v2/gzh360
+ * lib/v2/pku/nsd/gd
  * lib/v2/sdu/cs
  *
  * If your new route is not in the above folders, please add it to the list.

package/lib/v2/19lou/index.js

@@ -0,0 +1,74 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const iconv = require('iconv-lite');
+
+const setCookie = function (cookieName, cookieValue, seconds, path, domain, secure) {
+    let expires = null;
+    if (seconds !== -1) {
+        expires = new Date();
+        expires.setTime(expires.getTime() + seconds);
+    }
+    return [encodeURI(cookieName), '=', encodeURI(cookieValue), expires ? '; expires=' + expires.toGMTString() : '', path ? '; path=' + path : '/', domain ? '; domain=' + domain : '', secure ? '; secure' : ''].join('');
+};
+
+module.exports = async (ctx) => {
+    const city = ctx.params.city ?? 'www';
+
+    const rootUrl = `https://${city}.19lou.com`;
+
+    const response = await got({
+        method: 'get',
+        url: rootUrl,
+        responseType: 'buffer',
+    });
+
+    const $ = cheerio.load(iconv.decode(response.data, 'gbk'));
+
+    $('.title-more').remove();
+
+    let items = $('.center-center-jiazi')
+        .find('a[title]')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.attr('title'),
+                link: `https:${item.attr('href')}`,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                    responseType: 'buffer',
+                    headers: {
+                        cookie: setCookie('_Z3nY0d4C_', '37XgPK9h', 365, '/', '19lou.com'),
+                        referer: rootUrl,
+                    },
+                });
+
+                const content = cheerio.load(iconv.decode(detailResponse.data, 'gbk'));
+
+                content('.name-lz, .postView-pk-mod').remove();
+
+                item.author = content('.uname, .user-name').first().text();
+                item.description = content('.post-cont').first().html() || content('.thread-cont').html();
+                item.pubDate = timezone(parseDate(content('.cont-top-left meta').first().attr('content')), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text().split('-')[0],
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/19lou/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:city?': ['nczitzk'],
+};

package/lib/v2/19lou/radar.js

@@ -0,0 +1,29 @@
+module.exports = {
+    '19lou.com': {
+        _name: '19 楼',
+        '.': [
+            {
+                title: '头条',
+                docs: 'https://docs.rsshub.app/bbs.html#19-lou-tou-tiao',
+                source: ['/'],
+                target: (params, url) => `/19lou/${new URL(url).toString().match(/\/\/(.*?)\.19lou/)[1]}`,
+            },
+        ],
+        www: [
+            {
+                title: '头条',
+                docs: 'https://docs.rsshub.app/bbs.html#19-lou-tou-tiao',
+                source: ['/'],
+                target: '/19lou/www',
+            },
+        ],
+        jiaxing: [
+            {
+                title: '头条',
+                docs: 'https://docs.rsshub.app/bbs.html#19-lou-tou-tiao',
+                source: ['/'],
+                target: '/19lou/jiaxing',
+            },
+        ],
+    },
+};

package/lib/v2/19lou/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:city?', require('./index'));
+};

package/lib/v2/7mmtv/index.js

@@ -0,0 +1,86 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const language = ctx.params.language ?? 'en';
+    const category = ctx.params.category ?? 'censored_list';
+    const type = ctx.params.type ?? 'all';
+
+    const rootUrl = 'https://7mmtv.tv';
+    const currentUrl = `${rootUrl}/${language}/${category}/${type}/1.html`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    $('.pull-left p a').remove();
+
+    let items = $('.latest-korean-box-text a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const video = item.parent().parent().find('video');
+
+            return {
+                title: item.text(),
+                author: item.parent().find('.pull-left').text(),
+                pubDate: parseDate(item.parent().find('.date-part').text()),
+                link: `${item.attr('href').match(/^(http.*_content\/\d+)/)[1]}/index.html`,
+                poster: video.attr('poster'),
+                video: video.attr('srcmv'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    cover: content('.post-inner-details-img img').attr('src'),
+                    images: content('.video-introduction-images-list-row')
+                        .find('img')
+                        .toArray()
+                        .map((i) => content(i).attr('src')),
+                    poster: item.poster,
+                    video: item.video,
+                });
+
+                item.category = content('.actor-right-images-part p')
+                    .toArray()
+                    .map((a) => content(a).text())
+                    .concat(
+                        content('.posts-inner-details-text-under ul .posts-message span')
+                            .toArray()
+                            .map((c) => content(c).text())
+                    );
+
+                delete item.poster;
+                delete item.video;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title')
+            .text()
+            .replace(/ - Watch JAV Online/, ''),
+        link: currentUrl,
+        item: items,
+        description: $('meta[name="description"]').attr('content'),
+    };
+};

package/lib/v2/7mmtv/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/:language?/:category?/:type?': ['nczitzk'],
+    '/:language?/:category?/:id?': ['nczitzk'],
+};

package/lib/v2/7mmtv/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    '7mmtv.tv': {
+        _name: '7mmtv.tv',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/multimedia.html#7mmtv-fen-lei',
+                source: ['/'],
+                target: (params, url) => `/7mmtv/${new URL(url).toString().match(/\/(en|ja|ko|zh)\/([\w\d-]+\/){0,2}/)[1]}`,
+            },
+            {
+                title: '制作商',
+                docs: 'https://docs.rsshub.app/multimedia.html#7mmtv-zhi-zuo-shang',
+                source: ['/'],
+                target: (params, url) => `/7mmtv/${new URL(url).toString().match(/\/(en|ja|ko|zh)\/([\w\d-]+\/){0,2}/)[1]}`,
+            },
+        ],
+    },
+};

package/lib/v2/7mmtv/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:language?/:category?/:type?', require('./index'));
+};

package/lib/v2/7mmtv/templates/description.art

@@ -0,0 +1,11 @@
+{{ if cover }}
+<img src="{{ cover }}">
+{{ /if }}
+{{ if video }}
+<br>
+<video mute loop="loop" autoplay="autoplay" poster="{{ poster }}"><source src="{{ video }}"></video>
+<br>
+{{ /if }}
+{{ each images }}
+<img src="{{ $value }}">
+{{ /each }}

package/lib/v2/91porn/index.js

@@ -0,0 +1,58 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { domain = '91porn.com' } = ctx.query;
+    const siteUrl = `https://${domain}/index.php`;
+    const { lang = 'en_US' } = ctx.params;
+
+    const response = await got.post(siteUrl, {
+        form: {
+            session_language: lang,
+        },
+        headers: {
+            referer: siteUrl,
+        },
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.row .well')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.video-title').text(),
+                link: item.find('a').attr('href'),
+                poster: item.find('.img-responsive').attr('src'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(`91porn:${lang}:${new URL(item.link).searchParams.get('viewkey')}`, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+
+                item.pubDate = parseDate($('.title-yakov').eq(0).text(), 'YYYY-MM-DD');
+                item.description = art(path.join(__dirname, 'templates/index.art'), {
+                    link: item.link,
+                    poster: item.poster,
+                });
+                item.author = $('.title-yakov a span').text();
+                delete item.poster;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${$('.login_register_header').text()} - 91porn`,
+        link: siteUrl,
+        item: items,
+    };
+};

package/lib/v2/91porn/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:lang?': ['TonyRL'],
+};

package/lib/v2/91porn/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    '91porn.com': {
+        _name: '91porn',
+        '.': [
+            {
+                title: '91porn',
+                docs: 'https://docs.rsshub.app/multimedia.html#_91porn',
+                source: ['/index.php'],
+                target: '/91porn',
+            },
+        ],
+    },
+};

package/lib/v2/91porn/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:lang?', require('./index'));
+};

package/lib/v2/91porn/templates/index.art

@@ -0,0 +1 @@
+<a href="{{ link }}"><img src="{{ poster }} "></a>

package/lib/v2/9to5/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/9to5/:subsite/:tag?': ['HenryQW'],
+};

package/lib/v2/9to5/radar.js

@@ -0,0 +1,35 @@
+module.exports = {
+    '9to5toys.com': {
+        _name: '9to5',
+        '.': [
+            {
+                title: 'Toys 分站',
+                docs: 'https://docs.rsshub.app/new-media.html#_9to5',
+                source: ['/', '/guides/:tag'],
+                target: '/9to5/toys/:tag?',
+            },
+        ],
+    },
+    '9to5mac.com': {
+        _name: '9to5',
+        '.': [
+            {
+                title: 'Mac 分站',
+                docs: 'https://docs.rsshub.app/new-media.html#_9to5',
+                source: ['/', '/guides/:tag'],
+                target: '/9to5/mac/:tag?',
+            },
+        ],
+    },
+    '9to5google.com': {
+        _name: '9to5',
+        '.': [
+            {
+                title: 'Google 分站',
+                docs: 'https://docs.rsshub.app/new-media.html#_9to5',
+                source: ['/', '/guides/:tag'],
+                target: '/9to5/google/:tag?',
+            },
+        ],
+    },
+};

package/lib/v2/9to5/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:subsite/:tag?', require('./subsite'));
+};

package/lib/{routes → v2}/9to5/subsite.js

@@ -31,31 +31,34 @@ module.exports = async (ctx) => {
     }
 
     if (ctx.params.tag) {
-        link = `${link}/guides/${ctx.params.tag}/feed`;
+        link = `${link}/guides/${ctx.params.tag}/feed/`;
         title = `${ctx.params.tag} | ${title}`;
     } else {
-        link = `${link}/feed`;
+        link = `${link}/feed/`;
     }
 
     const feed = await parser.parseURL(link);
 
     const items = await Promise.all(
-        feed.items.splice(0, 10).map(async (item) => {
-            const response = await got({
-                method: 'get',
-                url: item.link,
-            });
-            const description = utils.ProcessFeed(response.data);
+        feed.items.splice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 10).map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const description = utils.ProcessFeed(response.data);
 
-            const single = {
-                title: item.title,
-                description,
-                pubDate: item.pubDate,
-                link: item.link,
-                author: item['dc:creator'],
-            };
-            return Promise.resolve(single);
-        })
+                const single = {
+                    title: item.title,
+                    description,
+                    pubDate: item.pubDate,
+                    link: item.link,
+                    author: item['dc:creator'],
+                };
+
+                return single;
+            })
+        )
     );
 
     ctx.state.data = {

package/lib/{routes → v2}/9to5/utils.js

@@ -2,7 +2,7 @@ const cheerio = require('cheerio');
 
 const ProcessFeed = (data) => {
     const $ = cheerio.load(data);
-    const content = $('div[itemprop="articleBody"]');
+    const content = $('div.post-body');
 
     const cover = $('meta[property="og:image"]');
     if (cover.length > 0) {
@@ -16,6 +16,9 @@ const ProcessFeed = (data) => {
         $(e).remove();
     });
 
+    // remove ad
+    content.find('div.ad-disclaimer-container').remove();
+
     content.find('div').each((i, e) => {
         if ($(e)[0].attribs.class) {
             const classes = $(e)[0].attribs.class;

package/lib/{routes → v2}/aliyun/database_month.js

@@ -10,7 +10,7 @@ module.exports = async (ctx) => {
         .map((i, e) => {
             const element = $(e);
             const title = element.find('a').text().trim();
-            const link = 'http://mysql.taobao.org' + element.find('a').attr('href').trim();
+            const link = `http://mysql.taobao.org${element.find('a').attr('href').trim()}/`;
             return {
                 title,
                 description: '',
@@ -20,20 +20,15 @@ module.exports = async (ctx) => {
         .get();
 
     const result = await Promise.all(
-        list.map(async (item) => {
+        list.map((item) => {
             const link = item.link;
 
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const itemReponse = await got.get(link);
-            const itemElement = cheerio.load(itemReponse.data);
-            item.description = itemElement('.content').html();
-
-            ctx.cache.set(link, JSON.stringify(item));
-            return item;
+            return ctx.cache.tryGet(link, async () => {
+                const itemReponse = await got(link);
+                const itemElement = cheerio.load(itemReponse.data);
+                item.description = itemElement('.content').html();
+                return item;
+            });
         })
     );
 

package/lib/{routes → v2}/aliyun/developer/group.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const type = ctx.params.type;
@@ -34,9 +35,13 @@ module.exports = async (ctx) => {
             list
                 .map((index, item) => {
                     item = $(item);
+                    const desc = item.find('.question-desc');
+                    const description = item.find('.browse').text() + ' ' + desc.find('.answer').text();
                     return {
-                        title: item.find('p').first().text().trim(),
+                        title: item.find('.question-title').text().trim() || item.find('a p').text().trim(),
                         link: item.find('a').attr('href'),
+                        pubDate: parseDate(item.find('.time').text()),
+                        description,
                     };
                 })
                 .get(),

package/lib/v2/aliyun/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/database_month': ['junbaor'],
+    '/developer/group/:type': ['umm233'],
+    '/notice/:type?': ['muzea'],
+};

package/lib/{routes → v2}/aliyun/notice.js

@@ -1,9 +1,7 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const dayjs = require('dayjs');
-const customParseFormat = require('dayjs/plugin/customParseFormat');
-
-dayjs.extend(customParseFormat);
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 const typeMap = {
     0: '9004748',
@@ -28,32 +26,26 @@ module.exports = async (ctx) => {
             const title = element.find('a').text().trim();
             const link = 'https://help.aliyun.com' + element.find('a').attr('href').trim();
             const date = element.find('.y-right').text();
-            const pubDate = dayjs(`${date} +0800`, 'YYYY-MM-DDHH:mm:ss ZZ');
+            const pubDate = timezone(parseDate(date), +8);
             return {
                 title,
                 description: '',
                 link,
-                pubDate: pubDate.toString(),
+                pubDate,
             };
         })
         .get();
 
     const result = await Promise.all(
-        list.map(async (item) => {
-            const link = item.link;
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const itemReponse = await got.get(link);
-            const itemElement = cheerio.load(itemReponse.data);
-            item.description = itemElement('#se-knowledge').html();
-
-            ctx.cache.set(link, JSON.stringify(item));
-            return item;
-        })
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const itemReponse = await got(item.link);
+                const itemElement = cheerio.load(itemReponse.data);
+                item.description = itemElement('#se-knowledge').html();
+
+                return item;
+            })
+        )
     );
 
     ctx.state.data = {