rsshub 1.0.0-master.f2ed98e → 1.0.0-master.f2eee10

package/lib/routes/everything/changes.js

@@ -9,7 +9,7 @@ module.exports = async (ctx) => {
     });
 
     const items = response.data
-        .replace(/\t\n|\t\n\n/g, '\n\n')
+        .replace(/\t\n\n|\t\n/g, '\n\n')
         .split('\n\n')
         .map((item) => {
             item = item.indexOf('\n') === 0 ? item.substr(2, item.length - 2) : item;

package/lib/routes/gov/hunan/notice.js

@@ -3,7 +3,7 @@ const got = require('@/utils/got');
 module.exports = async (ctx) => {
     const type = ctx.params.type || 'all';
     const rssData = {
-        isType: ctx.params.type ? true : false,
+        isType: !!ctx.params.type,
         Title: `湖南省政府采购网`,
         Desc: `湖南省政府采购网`,
         Link: `http://www.ccgp-hunan.gov.cn/page/notice/more.jsp`,

package/lib/utils/puppeteer.js

@@ -6,7 +6,7 @@ const logger = require('./logger');
 
 const options = {
     args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-infobars', '--window-position=0,0', '--ignore-certificate-errors', '--ignore-certificate-errors-spki-list', `--user-agent=${config.ua}`],
-    headless: true,
+    headless: 'new',
     ignoreHTTPSErrors: true,
 };
 
@@ -20,7 +20,7 @@ module.exports = async (extraOptions = {}) => {
         const { addExtra } = require('puppeteer-extra');
         puppeteer = addExtra(puppeteer);
 
-        // workaround for vercel/nft #54 and #283
+        // workaround for vercel/nft #54, #283, #304
         require('puppeteer-extra-plugin-stealth/evasions/chrome.app');
         require('puppeteer-extra-plugin-stealth/evasions/chrome.csi');
         require('puppeteer-extra-plugin-stealth/evasions/chrome.loadTimes');

package/lib/utils/render.js

@@ -1,53 +1,9 @@
 const art = require('art-template');
+const json = require('@/views/json');
 
 // We may add more control over it later
 
-/**
- * This function should be used by RSSHub middleware only.
- * @param {object} data ctx.state.data
- * @returns `JSON.stringify`-ed [JSON Feed](https://www.jsonfeed.org/)
- */
-const json = (data) => {
-    const jsonFeed = {
-        version: 'https://jsonfeed.org/version/1.1',
-        title: data.title || 'RSSHub',
-        home_page_url: data.link || 'https://docs.rsshub.app',
-        feed_url: data.feedLink,
-        description: `${data.description || data.title} - Made with love by RSSHub(https://github.com/DIYgod/RSSHub)`,
-        icon: data.image,
-        authors: typeof data.author === 'string' ? [{ name: data.author }] : data.author,
-        language: data.language || 'zh-cn',
-        items: data.item.map((item) => ({
-            id: item.guid || item.id || item.link,
-            url: item.link,
-            title: item.title,
-            content_html: (item.content && item.content.html) || item.description || item.title,
-            content_text: item.content && item.content.text,
-            image: item.image,
-            banner_image: item.banner,
-            date_published: item.pubDate,
-            date_modified: item.updated,
-            authors: typeof item.author === 'string' ? [{ name: item.author }] : item.author,
-            tags: typeof item.category === 'string' ? [item.category] : item.category,
-            language: item.language,
-            attachments: item.enclosure_url
-                ? [
-                      {
-                          url: item.enclosure_url,
-                          mime_type: item.enclosure_type,
-                          title: item.enclosure_title,
-                          size_in_bytes: item.enclosure_length,
-                          duration_in_seconds: item.itunes_duration,
-                      },
-                  ]
-                : undefined,
-            _extra: item._extra || undefined,
-        })),
-    };
-    return JSON.stringify(jsonFeed, null, 4);
-};
-
 module.exports = {
     art,
-    json,
+    json, // This should be used by RSSHub middleware only.
 };

package/lib/utils/request-wrapper.js

@@ -7,10 +7,10 @@ const https = require('https');
 let agent = null;
 if (proxyUri) {
     if (proxyUri.startsWith('http')) {
-        const HttpsProxyAgent = require('https-proxy-agent');
+        const { HttpsProxyAgent } = require('https-proxy-agent');
         agent = new HttpsProxyAgent(proxyUri);
     } else if (proxyUri.startsWith('socks')) {
-        const SocksProxyAgent = require('socks-proxy-agent').SocksProxyAgent;
+        const { SocksProxyAgent } = require('socks-proxy-agent');
         agent = new SocksProxyAgent(proxyUri);
     }
 }

package/lib/utils/wechat-mp.js

@@ -213,7 +213,7 @@ const fetchArticle = (ctx, url, bypassHostCheck = false) => {
         }
 
         let pubDate;
-        const publish_time_script = $('script[nonce][type="text/javascript"]:contains("var ct")').first().html();
+        const publish_time_script = $('script[nonce][type="text/javascript"]:contains("var ct")').text();
         const publish_time_match = publish_time_script && publish_time_script.match(/var ct *= *"?(\d{10})"?/);
         const publish_timestamp = publish_time_match && publish_time_match[1];
         if (publish_timestamp) {
@@ -222,7 +222,7 @@ const fetchArticle = (ctx, url, bypassHostCheck = false) => {
 
         let mpName = $('.profile_nickname').first().text();
         mpName = mpName && mpName.trim();
-        return { title, author, description, summary, pubDate, mpName, link: url };
+        return { title, author, description, summary, pubDate, mpName, link: response.url };
     });
 };
 

package/lib/v2/0818tuan/index.js

@@ -0,0 +1,47 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const baseUrl = 'http://www.0818tuan.com';
+    const { listId = '1' } = ctx.params;
+    const url = `${baseUrl}/list-${listId}-0.html`;
+
+    const { data: response } = await got(url);
+    const $ = cheerio.load(response);
+
+    const list = $(listId === '3' ? '.col-xs-12 .thumbnail > a' : '.col-md-8 .list-group > a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.attr('title'),
+                link: `${baseUrl}${item.attr('href')}`,
+            };
+        })
+        .filter((i) => !i.link.includes('m.0818tuan.com/tb1111.php'));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+
+                $('.pageLink, .alert, p[style="margin:15px;"]').remove();
+
+                item.description = $('.post-content').html();
+                item.pubDate = timezone(parseDate($('.panel-body > .text-center').text().replace('时间:', ''), 'YYYY-MM-DD HH:mm:ss'), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        image: 'http://www.0818tuan.com/favicon.ico',
+        item: items,
+    };
+};

package/lib/v2/0818tuan/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:listId?': ['TonyRL'],
+};

package/lib/v2/0818tuan/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    '0818tuan.com': {
+        _name: '0818 团',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/shopping.html#_0818-tuan',
+                source: ['/:listId', '/'],
+                target: (params) => `/0818tuan${params.listId ? '/' + params.listId.replace('list-', '').replace('-0.html', '') : ''}`,
+            },
+        ],
+    },
+};

package/lib/v2/0818tuan/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:listId?', require('./index'));
+};

package/lib/v2/141jav/index.js

@@ -0,0 +1,71 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.141jav.com';
+    const currentUrl = `${rootUrl}${ctx.path}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    if (ctx.path === '/') {
+        ctx.redirect(`/141jav${$('.overview').first().attr('href')}`);
+        return;
+    }
+
+    const items = $('.columns')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const id = item.find('.title a').text();
+            const size = item.find('.title span').text();
+            const pubDate = item.find('.subtitle a').attr('href').split('/date/').pop();
+            const description = item.find('.has-text-grey-dark').text();
+            const actresses = item
+                .find('.panel-block')
+                .toArray()
+                .map((a) => $(a).text().trim());
+            const tags = item
+                .find('.tag')
+                .toArray()
+                .map((t) => $(t).text().trim());
+            const magnet = item.find('a[title="Magnet torrent"]').attr('href');
+            const link = item.find('a[title="Download .torrent"]').attr('href');
+            const image = item.find('.image').attr('src');
+
+            return {
+                title: `${id} ${size}`,
+                pubDate: parseDate(pubDate, 'YYYY/MM/DD'),
+                link: new URL(item.find('a').first().attr('href'), rootUrl).href,
+                description: art(path.join(__dirname, 'templates/description.art'), {
+                    image,
+                    id,
+                    size,
+                    pubDate,
+                    description,
+                    actresses,
+                    tags,
+                    magnet,
+                    link,
+                }),
+                author: actresses.join(', '),
+                category: [...tags, ...actresses],
+                enclosure_type: 'application/x-bittorrent',
+                enclosure_url: magnet,
+            };
+        });
+
+    ctx.state.data = {
+        title: `141JAV - ${$('title').text().split('-')[0].trim()}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/141jav/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:type?/:keyword?': ['cgkings', 'nczitzk'],
+};

package/lib/v2/141jav/radar.js

@@ -0,0 +1,40 @@
+module.exports = {
+    '141jav.com': {
+        _name: '141JAV',
+        '.': [
+            {
+                title: '今日种子',
+                docs: 'https://docs.rsshub.app/multimedia.html#_141jav',
+                source: '/',
+                target: (params, url, document) => {
+                    const today = document.querySelector('div.card.mb-1.card-overview').getAttribute('data-date').replace(/-/g, '');
+                    return `/141jav/day/${today}`;
+                },
+            },
+            {
+                title: '今日演员',
+                docs: 'https://docs.rsshub.app/multimedia.html#_141jav',
+                source: '/',
+                target: (params, url, document) => {
+                    const star = document.querySelector('div.card-content > div > a').getAttribute('href');
+                    return `/141jav${star}`;
+                },
+            },
+            {
+                title: '页面种子',
+                docs: 'https://docs.rsshub.app/multimedia.html#_141jav',
+                source: ['/:type', '/:type/:key', '/:type/:key/:morekey'],
+                target: (params, url, document) => {
+                    const itype = params.morekey === undefined ? params.type : params.type === 'tag' ? 'tag' : 'day';
+                    let ikey = `${itype === 'day' ? params.type : ''}${params.key || ''}${itype === 'tag' && params.morekey !== undefined ? '%2F' : ''}${params.morekey || ''}`;
+                    if (ikey === '' && itype === 'tag') {
+                        ikey = document.querySelector('div.thumbnail.is-inline > a').getAttribute('href').replace('/tag/', '').replace(/\//g, '%2F');
+                    } else if (ikey === '' && itype === 'actress') {
+                        ikey = document.querySelector('div.card > a').getAttribute('href').replace('/actress/', '');
+                    }
+                    return `/141jav/${itype}/${ikey}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/141jav/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get(/([\w-/]+)?/, require('./index'));
+};

package/lib/v2/141jav/templates/description.art

@@ -0,0 +1,47 @@
+{{ if image }}
+<img src="{{ image }}">
+{{ /if }}
+<table>
+  <tbody>
+    <tr>
+      <th>ID</th>
+      <td>{{ id }}</td>
+    </tr>
+    <tr>
+      <th>Size</th>
+      <td>{{ size }}</td>
+    </tr>
+    <tr>
+      <th>Date</th>
+      <td>{{ pubDate }}</td>
+    </tr>
+    <tr>
+      <th>Description</th>
+      <td>{{ description }}</td>
+    </tr>
+    <tr>
+      <th>Actress</th>
+      <td>
+        {{ each actresses actress }}
+          <a href="/actress/{{ actress }}">{{ actress }}</a>&nbsp
+        {{ /each }}
+      </td>
+    </tr>
+    <tr>
+      <th>Tag</th>
+      <td>
+        {{ each tags tag }}
+          <a href="/tag/{{ tag }}">{{ tag }}</a>&nbsp
+        {{ /each }}
+      </td>
+    </tr>
+    <tr>
+      <th>Magnet torrent</th>
+      <td><a href="{{ magnet }}">Magnet torrent link</a></td>
+    </tr>
+    <tr>
+      <th>Download .torrent</th>
+      <td><a href="{{ link }}">Download torrent</a></td>
+    </tr>
+  </tbody>
+</table>

package/lib/v2/141ppv/index.js

@@ -0,0 +1,71 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.141ppv.com';
+    const currentUrl = `${rootUrl}${ctx.path}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    if (ctx.path === '/') {
+        ctx.redirect(`/141ppv${$('.overview').first().attr('href')}`);
+        return;
+    }
+
+    const items = $('.columns')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const id = item.find('.title a').text();
+            const size = item.find('.title span').text();
+            const pubDate = item.find('.subtitle a').attr('href').split('/date/').pop();
+            const description = item.find('.has-text-grey-dark').text();
+            const actresses = item
+                .find('.panel-block')
+                .toArray()
+                .map((a) => $(a).text().trim());
+            const tags = item
+                .find('.tag')
+                .toArray()
+                .map((t) => $(t).text().trim());
+            const magnet = item.find('a[title="Magnet torrent"]').attr('href');
+            const link = item.find('a[title="Download .torrent"]').attr('href');
+            const image = item.find('.image').attr('src');
+
+            return {
+                title: `${id} ${size}`,
+                pubDate: parseDate(pubDate, 'YYYY/MM/DD'),
+                link: new URL(item.find('a').first().attr('href'), rootUrl).href,
+                description: art(path.join(__dirname, 'templates/description.art'), {
+                    image,
+                    id,
+                    size,
+                    pubDate,
+                    description,
+                    actresses,
+                    tags,
+                    magnet,
+                    link,
+                }),
+                author: actresses.join(', '),
+                category: [...tags, ...actresses],
+                enclosure_type: 'application/x-bittorrent',
+                enclosure_url: magnet,
+            };
+        });
+
+    ctx.state.data = {
+        title: `141PPV - ${$('title').text().split('-')[0].trim()}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/141ppv/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:type?/:keyword?': ['cgkings', 'nczitzk'],
+};

package/lib/v2/141ppv/radar.js

@@ -0,0 +1,40 @@
+module.exports = {
+    '141ppv.com': {
+        _name: '141JAV',
+        '.': [
+            {
+                title: '今日种子',
+                docs: 'https://docs.rsshub.app/multimedia.html#_141ppv',
+                source: '/',
+                target: (params, url, document) => {
+                    const today = document.querySelector('div.card.mb-1.card-overview').getAttribute('data-date').replace(/-/g, '');
+                    return `/141ppv/day/${today}`;
+                },
+            },
+            {
+                title: '今日演员',
+                docs: 'https://docs.rsshub.app/multimedia.html#_141ppv',
+                source: '/',
+                target: (params, url, document) => {
+                    const star = document.querySelector('div.card-content > div > a').getAttribute('href');
+                    return `/141ppv${star}`;
+                },
+            },
+            {
+                title: '页面种子',
+                docs: 'https://docs.rsshub.app/multimedia.html#_141ppv',
+                source: ['/:type', '/:type/:key', '/:type/:key/:morekey'],
+                target: (params, url, document) => {
+                    const itype = params.morekey === undefined ? params.type : params.type === 'tag' ? 'tag' : 'day';
+                    let ikey = `${itype === 'day' ? params.type : ''}${params.key || ''}${itype === 'tag' && params.morekey !== undefined ? '%2F' : ''}${params.morekey || ''}`;
+                    if (ikey === '' && itype === 'tag') {
+                        ikey = document.querySelector('div.thumbnail.is-inline > a').getAttribute('href').replace('/tag/', '').replace(/\//g, '%2F');
+                    } else if (ikey === '' && itype === 'actress') {
+                        ikey = document.querySelector('div.card > a').getAttribute('href').replace('/actress/', '');
+                    }
+                    return `/141ppv/${itype}/${ikey}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/141ppv/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get(/([\w-/]+)?/, require('./index'));
+};

package/lib/v2/141ppv/templates/description.art

@@ -0,0 +1,47 @@
+{{ if image }}
+<img src="{{ image }}">
+{{ /if }}
+<table>
+  <tbody>
+    <tr>
+      <th>ID</th>
+      <td>{{ id }}</td>
+    </tr>
+    <tr>
+      <th>Size</th>
+      <td>{{ size }}</td>
+    </tr>
+    <tr>
+      <th>Date</th>
+      <td>{{ pubDate }}</td>
+    </tr>
+    <tr>
+      <th>Description</th>
+      <td>{{ description }}</td>
+    </tr>
+    <tr>
+      <th>Actress</th>
+      <td>
+        {{ each actresses actress }}
+          <a href="/actress/{{ actress }}">{{ actress }}</a>&nbsp
+        {{ /each }}
+      </td>
+    </tr>
+    <tr>
+      <th>Tag</th>
+      <td>
+        {{ each tags tag }}
+          <a href="/tag/{{ tag }}">{{ tag }}</a>&nbsp
+        {{ /each }}
+      </td>
+    </tr>
+    <tr>
+      <th>Magnet torrent</th>
+      <td><a href="{{ magnet }}">Magnet torrent link</a></td>
+    </tr>
+    <tr>
+      <th>Download .torrent</th>
+      <td><a href="{{ link }}">Download torrent</a></td>
+    </tr>
+  </tbody>
+</table>

package/lib/v2/163/maintainer.js

@@ -5,6 +5,7 @@ module.exports = {
     '/dy2/:id': ['mjysci'],
 
     '/music/artist/:id': ['metowolf'],
+    '/music/artist/songs/:id': ['ZhongMingKun'],
     '/music/djradio/:id': ['magic-akari'],
     '/music/playlist/:id': ['DIYgod'],
     '/music/user/events/:id': ['Master-Hash'],

package/lib/v2/163/music/artist-songs.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+
+    const { data } = await got(`https://music.163.com/api/v1/artist/songs`, {
+        headers: {
+            Referer: 'https://music.163.com/',
+        },
+        searchParams: {
+            id,
+            private_cloud: 'true',
+            work_type: 1,
+            order: 'time',
+            offset: 0,
+            limit: 100,
+        },
+    });
+
+    const artist = data.songs.find(({ ar }) => ar[0].id === parseInt(id)).ar[0];
+    const items = data.songs.map((song) => ({
+        title: `${song.name} - ${song.ar.map(({ name }) => name).join(' / ')}`,
+        description: art(path.join(__dirname, '../templates/music/playlist.art'), {
+            singer: song.ar.map(({ name }) => name).join(' / '),
+            album: song.al.name,
+            picUrl: song.al.picUrl,
+        }),
+        link: `https://music.163.com/#/song?id=${song.id}`,
+    }));
+
+    ctx.state.data = {
+        title: `${artist.name} - 歌手歌曲`,
+        link: `https://music.163.com/#/artist?id=${id}`,
+        description: `网易云音乐 - 歌手歌曲 - ${artist.name}`,
+        item: items,
+    };
+};

package/lib/v2/163/radar.js

@@ -63,6 +63,15 @@ module.exports = {
                     return id ? `/163/music/artist/${id}` : '';
                 },
             },
+            {
+                title: '云音乐 - 歌手歌曲',
+                docs: 'https://docs.rsshub.app/multimedia.html#wang-yi-yun-yin-yue',
+                source: '/',
+                target: (_params, url) => {
+                    const id = new URL(url).hash.match(/artist\?id=(.*)/)[1];
+                    return id ? `/163/music/artist/songs/${id}` : '';
+                },
+            },
             {
                 title: '云音乐 - 电台节目',
                 docs: 'https://docs.rsshub.app/multimedia.html#wang-yi-yun-yin-yue',
@@ -117,7 +126,7 @@ module.exports = {
                 title: '云音乐 - 用户歌单',
                 docs: 'https://docs.rsshub.app/multimedia.html#wang-yi-yun-yin-yue',
                 source: '/m/user',
-                target: (params, url) => `/163/music/playlist/${new URL(url).searchParams.get('id')}`,
+                target: (params, url) => `/163/music/user/playlist/${new URL(url).searchParams.get('id')}`,
             },
             {
                 title: '云音乐 - 歌单歌曲',
@@ -129,13 +138,13 @@ module.exports = {
                 title: '云音乐 - 歌手专辑',
                 docs: 'https://docs.rsshub.app/multimedia.html#wang-yi-yun-yin-yue',
                 source: '/m/album',
-                target: (params, url) => `/163/music/playlist/${new URL(url).searchParams.get('id')}`,
+                target: (params, url) => `/163/music/artist/${new URL(url).searchParams.get('id')}`,
             },
             {
-                title: '云音乐 - 播单声音',
+                title: '云音乐 - 电台节目',
                 docs: 'https://docs.rsshub.app/multimedia.html#wang-yi-yun-yin-yue',
                 source: ['/m/radio', '/m/djradio'],
-                target: (params, url) => `/163/music/playlist/${new URL(url).searchParams.get('id')}`,
+                target: (params, url) => `/163/music/djradio/${new URL(url).searchParams.get('id')}`,
             },
             {
                 title: '用户动态',