rsshub 1.0.0-master.fb2a407 → 1.0.0-master.fc5a2f6

package/lib/{routes → v2}/youtube/user.js

@@ -1,5 +1,6 @@
 const utils = require('./utils');
 const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     if (!config.youtube || !config.youtube.key) {
@@ -10,7 +11,7 @@ module.exports = async (ctx) => {
 
     const playlistId = (await utils.getChannelWithUsername(username, 'contentDetails', ctx.cache)).data.items[0].contentDetails.relatedPlaylists.uploads;
 
-    const data = (await utils.getPlaylistItems(playlistId, 'snippet,contentDetails')).data.items;
+    const data = (await utils.getPlaylistItems(playlistId, 'snippet', ctx.cache)).data.items;
 
     ctx.state.data = {
         title: `${username} - YouTube`,
@@ -24,10 +25,8 @@ module.exports = async (ctx) => {
                 const img = utils.getThumbnail(snippet.thumbnails);
                 return {
                     title: snippet.title,
-                    description: `${
-                        embed ? '<iframe id="ytplayer" type="text/html" width="640" height="360" src="https://www.youtube-nocookie.com/embed/' + videoId + '" frameborder="0" allowfullscreen></iframe>' : '<img src="' + img.url + '">'
-                    }${utils.formatDescription(snippet.description)}`,
-                    pubDate: new Date(snippet.publishedAt).toUTCString(),
+                    description: utils.renderDescription(embed, videoId, img, utils.formatDescription(snippet.description)),
+                    pubDate: parseDate(snippet.publishedAt),
                     link: `https://www.youtube.com/watch?v=${videoId}`,
                 };
             }),

package/lib/v2/youtube/utils.js

@@ -0,0 +1,141 @@
+const { google } = require('googleapis');
+const { OAuth2 } = google.auth;
+const { art } = require('@/utils/render');
+const path = require('path');
+const config = require('@/config').value;
+
+let count = 0;
+const youtube = {};
+if (config.youtube && config.youtube.key) {
+    const keys = config.youtube.key.split(',');
+
+    keys.forEach((key, index) => {
+        if (key) {
+            youtube[index] = google.youtube({
+                version: 'v3',
+                auth: key,
+            });
+            count = index + 1;
+        }
+    });
+}
+
+let index = -1;
+const exec = async (func) => {
+    let result;
+    for (let i = 0; i < count; i++) {
+        index++;
+        try {
+            // eslint-disable-next-line no-await-in-loop
+            result = await func(youtube[index % count]);
+            break;
+        } catch (error) {
+            // console.error(error);
+        }
+    }
+    return result;
+};
+
+let youtubeOAuth2Client;
+if (config.youtube && config.youtube.clientId && config.youtube.clientSecret && config.youtube.refreshToken) {
+    youtubeOAuth2Client = new OAuth2(config.youtube.clientId, config.youtube.clientSecret, 'https://developers.google.com/oauthplayground');
+    youtubeOAuth2Client.setCredentials({ refresh_token: config.youtube.refreshToken });
+}
+
+const youtubeUtils = {
+    getPlaylistItems: (id, part, cache) =>
+        cache.tryGet(
+            `youtube:getPlaylistItems:${id}`,
+            async () => {
+                const res = await exec((youtube) =>
+                    youtube.playlistItems.list({
+                        part,
+                        playlistId: id,
+                        maxResults: 50, // youtube api param value default is 5
+                    })
+                );
+                return res;
+            },
+            config.cache.routeExpire,
+            false
+        ),
+    getPlaylist: (id, part, cache) =>
+        cache.tryGet(`youtube:getPlaylist:${id}`, async () => {
+            const res = await exec((youtube) =>
+                youtube.playlists.list({
+                    part,
+                    id,
+                })
+            );
+            return res;
+        }),
+    getChannelWithId: (id, part, cache) =>
+        cache.tryGet(`youtube:getChannelWithId:${id}`, async () => {
+            const res = await exec((youtube) =>
+                youtube.channels.list({
+                    part,
+                    id,
+                })
+            );
+            return res;
+        }),
+    getChannelWithUsername: (username, part, cache) =>
+        cache.tryGet(`youtube:getChannelWithUsername:${username}`, async () => {
+            const res = await exec((youtube) =>
+                youtube.channels.list({
+                    part,
+                    forUsername: username,
+                })
+            );
+            return res;
+        }),
+    // not in use
+    // getVideoAuthor: async (id, part) => {
+    //     const res = await exec((youtube) =>
+    //         youtube.videos.list({
+    //             part,
+    //             id,
+    //         })
+    //     );
+    //     return res;
+    // },
+    getThumbnail: (thumbnails) => thumbnails.maxres || thumbnails.standard || thumbnails.high || thumbnails.medium || thumbnails.default,
+    formatDescription: (description) => description.replace(/(?:\r\n|\r|\n)/g, '<br>'),
+    renderDescription: (embed, videoId, img, description) =>
+        art(path.join(__dirname, 'templates/description.art'), {
+            embed,
+            videoId,
+            img,
+            description,
+        }),
+
+    getSubscriptions: async (part, cache) => {
+        // access tokens expire after one hour
+        let accessToken = await cache.get('youtube:accessToken', false);
+        if (!accessToken) {
+            const data = await youtubeOAuth2Client.getAccessToken();
+            accessToken = data.token;
+            await cache.set('youtube:accessToken', accessToken, ~~((data.res.data.expiry_date - Date.now()) / 1000)); // ~3600s
+        }
+        youtubeOAuth2Client.setCredentials({ access_token: accessToken, refresh_token: config.youtube.refreshToken });
+
+        return cache.tryGet('youtube:getSubscriptions', () => youtubeUtils.getSubscriptionsRecusive(part), config.cache.routeExpire, false);
+    },
+    getSubscriptionsRecusive: async (part, nextPageToken = undefined) => {
+        const res = await google.youtube('v3').subscriptions.list({
+            auth: youtubeOAuth2Client,
+            part,
+            mine: true,
+            maxResults: 50,
+            pageToken: nextPageToken ? nextPageToken : undefined,
+        });
+        // recursively get next page
+        if (res.data.nextPageToken) {
+            const next = await youtubeUtils.getSubscriptionsRecusive(part, res.data.nextPageToken);
+            res.data.items = [...res.data.items, ...next.data.items];
+        }
+        return res;
+    },
+};
+
+module.exports = youtubeUtils;

package/lib/v2/youzhiyouxing/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/materials/:column?': ['broven', 'Fatpandac'],
+    '/materials/:id?': ['broven', 'Fatpandac', 'nczitzk'],
 };

package/lib/v2/youzhiyouxing/materials.js

@@ -2,35 +2,46 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 
-const rootUrl = 'https://youzhiyouxing.cn';
-
 module.exports = async (ctx) => {
-    const column = ctx.params.column ?? '0';
-    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 30;
-    const url = `${rootUrl}/materials?column=${column}&limit=${limit}`;
+    const id = ctx.params.id ?? '';
+
+    const rootUrl = 'https://youzhiyouxing.cn';
+    const currentUrl = `${rootUrl}/materials?column_id=${id}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
 
-    const response = await got(url);
     const $ = cheerio.load(response.data);
-    const title = $(`a[phx-value-column="${column}"]`).text() === '' ? '全部' : $(`a[phx-value-column="${column}"]`).text();
-    const list = $('[phx-click=material]')
-        .map((_, item) => ({
-            title: $(item).find('h3').text(),
-            link: `${rootUrl}/materials/${$(item).attr('phx-value-id')}`,
-            pubDate: parseDate($(item).find('div.xs-text.muted-text.tw-text-right').text(), ['YYYY年M月D日', 'M月D日']),
-        }))
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
+
+    let items = $('li[phx-click="material"]')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: `${rootUrl}/materials/${item.attr('phx-value-id')}`,
+                pubDate: parseDate(item.find('.tw-text-right').text(), ['YYYY年M月D日', 'M月D日']),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got(item.link);
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
                 const content = cheerio.load(detailResponse.data);
 
+                item.author = content('.tw-inline').text().replace('·', '');
                 item.description = content('#zx-rangy-root')
                     .html()
                     .replace(/(<img.*?) src(=.*?>)/g, '$1 data$2')
                     .replace(/(<img.*?) data-src(=.*?>)/g, '$1 src$2');
-                item.author = content('span.tw-inline').text().replace('·', '');
 
                 return item;
             })
@@ -38,9 +49,8 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: `有知有行 - ${title}`,
-        link: url,
-        description: `有知有行 - ${title}`,
+        title: `有知有行 - ${$(`a[phx-value-column_id="${id}"]`).text()}`,
+        link: currentUrl,
         item: items,
     };
 };

package/lib/v2/youzhiyouxing/radar.js

@@ -13,7 +13,7 @@ module.exports = {
                 docs: 'https://docs.rsshub.app/finance.html#you-you-wei-zhi-zhi-you-you-wei-xing-hang-xing-hang-heng-you-you-wei-zhi-zhi-wen-zhang-zhang',
                 source: ['/materials'],
                 target: (_params, url) => {
-                    if (new URL(url).searchParams.get('column') === '4') {
+                    if (new URL(url).searchParams.get('column_id') === '4') {
                         return '/youzhiyouxing/materials/4';
                     }
                 },
@@ -23,7 +23,7 @@ module.exports = {
                 docs: 'https://docs.rsshub.app/finance.html#you-you-wei-zhi-zhi-you-you-wei-xing-hang-xing-hang-heng-you-you-wei-zhi-zhi-wen-zhang-zhang',
                 source: ['/materials'],
                 target: (_params, url) => {
-                    if (new URL(url).searchParams.get('column') === '2') {
+                    if (new URL(url).searchParams.get('column_id') === '2') {
                         return '/youzhiyouxing/materials/2';
                     }
                 },
@@ -33,7 +33,7 @@ module.exports = {
                 docs: 'https://docs.rsshub.app/finance.html#you-you-wei-zhi-zhi-you-you-wei-xing-hang-xing-hang-heng-you-you-wei-zhi-zhi-wen-zhang-zhang',
                 source: ['/materials'],
                 target: (_params, url) => {
-                    if (new URL(url).searchParams.get('column') === '10') {
+                    if (new URL(url).searchParams.get('column_id') === '10') {
                         return '/youzhiyouxing/materials/10';
                     }
                 },
@@ -43,7 +43,7 @@ module.exports = {
                 docs: 'https://docs.rsshub.app/finance.html#you-you-wei-zhi-zhi-you-you-wei-xing-hang-xing-hang-heng-you-you-wei-zhi-zhi-wen-zhang-zhang',
                 source: ['/materials'],
                 target: (_params, url) => {
-                    if (new URL(url).searchParams.get('column') === '1') {
+                    if (new URL(url).searchParams.get('column_id') === '1') {
                         return '/youzhiyouxing/materials/1';
                     }
                 },
@@ -53,7 +53,7 @@ module.exports = {
                 docs: 'https://docs.rsshub.app/finance.html#you-you-wei-zhi-zhi-you-you-wei-xing-hang-xing-hang-heng-you-you-wei-zhi-zhi-wen-zhang-zhang',
                 source: ['/materials'],
                 target: (_params, url) => {
-                    if (new URL(url).searchParams.get('column') === '3') {
+                    if (new URL(url).searchParams.get('column_id') === '3') {
                         return '/youzhiyouxing/materials/3';
                     }
                 },
@@ -63,7 +63,7 @@ module.exports = {
                 docs: 'https://docs.rsshub.app/finance.html#you-you-wei-zhi-zhi-you-you-wei-xing-hang-xing-hang-heng-you-you-wei-zhi-zhi-wen-zhang-zhang',
                 source: ['/materials'],
                 target: (_params, url) => {
-                    if (new URL(url).searchParams.get('column') === '11') {
+                    if (new URL(url).searchParams.get('column_id') === '11') {
                         return '/youzhiyouxing/materials/11';
                     }
                 },

package/lib/v2/youzhiyouxing/router.js

@@ -1,3 +1,3 @@
 module.exports = function (router) {
-    router.get('/materials/:column?', require('./materials'));
+    router.get('/materials/:id?', require('./materials'));
 };

package/lib/v2/yunspe/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/newsflash': ['x2009again'],
+};

package/lib/v2/yunspe/newsflash.js

@@ -0,0 +1,42 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const response = await got({
+        method: 'post',
+        url: 'https://www.yunspe.com/wp-json/b2/v1/getNewsflashesList',
+        headers: {
+            Referer: 'https://www.yunspe.com/newsflashes',
+            'content-type': 'application/x-www-form-urlencoded',
+        },
+        body: 'paged=1&term=0&post_paged=1',
+    });
+    const data = response.data.data;
+
+    const out = data.map((item) => {
+        const title = item[0].title.replace('【微语简报】', '');
+        const link = item[0].link;
+        const datematch = item[0]._date.match(/datetime="(.*)" itemprop/);
+        const date = datematch && datematch[1];
+        const description = item[0].content
+            .substring(item[0].content.indexOf('【今日简报】') + 6)
+            .replace(/(\d*\.\s)|(【)/g, '<br>$1$2')
+            .substring(4);
+
+        const single = {
+            title,
+            link,
+            pubDate: date && timezone(parseDate(date, 'YYYY-M-D H:mm:ss'), +8),
+            description,
+        };
+
+        return single;
+    });
+    ctx.state.data = {
+        title: '微语简报',
+        link: 'https://www.yunspe.com/newsflashes',
+        description: '微语简报',
+        item: out,
+    };
+};

package/lib/v2/yunspe/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'yunspe.com': {
+        _name: '云奇网',
+        '.': [
+            {
+                title: '微语简报',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-yu-jian-bao',
+                source: ['/newsflashes/微语简报', '/newsflashes', '/'],
+                target: '/yunspe/newsflashes',
+            },
+        ],
+    },
+};

package/lib/v2/yunspe/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/newsflash', require('./newsflash'));
+};

package/lib/v2/yyets/article.js

@@ -0,0 +1,45 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const baseURL = 'https://yysub.net';
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type ?? '';
+    const url = `${baseURL}/article${type ? '?type=' + type : ''}`;
+
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+
+    let items = $('.article-list li .fl-info')
+        .toArray()
+        .map((e) => {
+            e = $(e);
+            return {
+                title: e.find('h3 a').text(),
+                link: `${baseURL}${e.find('h3 a').attr('href')}`,
+                author: e.find('p a').text(),
+                pubDate: timezone(parseDate(e.find('p').eq(2).text()), +8),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('.information-desc').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${$('title').text()} - 人人影视`,
+        description: $('meta[name="description"]').attr('content'),
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/yyets/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/article/:type?': ['wb121017405'],
+    '/today': ['bao1991213'],
+};

package/lib/v2/yyets/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'yysub.net': {
+        _name: '人人影视',
+        '.': [
+            {
+                title: '影视资讯',
+                docs: 'https://docs.rsshub.app/multimedia#ren-ren-ying-shi',
+                source: '/article',
+                target: (_params, url) => `/yyets/article${new URL(url).searchParams.has('type') ? '/' + new URL(url).searchParams.get('type') : ''}`,
+            },
+            {
+                title: '今日播出',
+                docs: 'https://docs.rsshub.app/multimedia#ren-ren-ying-shi',
+                source: ['/tv/schedule', '/'],
+                target: '/yyets/today',
+            },
+        ],
+    },
+};

package/lib/v2/yyets/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/article/:type?', require('./article'));
+    router.get('/today', require('./today'));
+};

package/lib/v2/yyets/today.js

@@ -0,0 +1,37 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    // 发起 HTTP GET 请求
+    const response = await got({
+        method: 'get',
+        url: 'https://yysub.net',
+    });
+
+    const data = response.data; // response.data 为 HTTP GET 请求返回的 HTML，也就是简书首页的所有 HTML
+
+    const $ = cheerio.load(data); // 使用 cheerio 加载返回的 HTML
+    const list = $('.today-list-wrap').find('ul').find('li');
+    // 使用 cheerio 选择器，选择带有 data-item_id 属性的所有 div 元素，返回 cheerio node 对象数组
+
+    // 注：每一个 cheerio node 对应一个 HTML DOM
+    // 注：cheerio 选择器与 jquery 选择器几乎相同
+    // 参考 cheerio 文档：https://cheerio.js.org/
+
+    ctx.state.data = {
+        title: '人人影视-今日播出',
+        link: 'https://yysub.net',
+        item:
+            list &&
+            list
+                .map((index, item) => {
+                    item = $(item);
+                    return {
+                        title: item.find('a').first().text(),
+                        link: item.find('a').attr('href'),
+                        guid: item.find('a').first().text(),
+                    };
+                })
+                .get(),
+    };
+};

package/lib/{routes → v2}/yystv/category.js

@@ -1,10 +1,10 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const date = require('@/utils/date');
+const { parseDate, parseRelativeDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const category = ctx.params.category;
-    const url = `http://www.yystv.cn/b/${category}`;
+    const url = `https://www.yystv.cn/b/${category}`;
     const response = await got({
         method: 'get',
         url,
@@ -18,8 +18,8 @@ module.exports = async (ctx) => {
         .map(function () {
             const info = {
                 title: $(this).find('.b-main-info-title').text(),
-                link: 'http://www.yystv.cn' + $(this).find('.b-main-info-title a').attr('href'),
-                pubDate: date($(this).find('.b-main-createtime').text()),
+                link: 'https://www.yystv.cn' + $(this).find('.b-main-info-title a').attr('href'),
+                pubDate: parseRelativeDate($(this).find('.b-main-createtime').text()),
                 author: $(this).find('.b-author').text(),
             };
             return info;
@@ -31,8 +31,8 @@ module.exports = async (ctx) => {
         .map(function () {
             const info = {
                 title: $('.list-article-title', this).text(),
-                link: 'http://www.yystv.cn' + $('a', this).attr('href'),
-                pubDate: date($('.c-999', this).text()),
+                link: 'https://www.yystv.cn' + $('a', this).attr('href'),
+                pubDate: $('.c-999', this).text().includes('-') ? parseDate($('.c-999', this).text()) : parseRelativeDate($('.c-999', this).text()),
                 author: $('.handler-author-link', this).text(),
             };
             return info;
@@ -49,7 +49,7 @@ module.exports = async (ctx) => {
                         method: 'get',
                     });
                     const $ = cheerio.load(r.data);
-                    return $('.doc-content div').html();
+                    return $('.doc-content.rel').html();
                 });
                 return currentValue;
             })
@@ -58,7 +58,7 @@ module.exports = async (ctx) => {
     await getDescription(items).then(() => {
         ctx.state.data = {
             title: '游研社-' + $('title').text(),
-            link: `http://www.yystv.cn/b/${category}`,
+            link: `https://www.yystv.cn/b/${category}`,
             item: items,
         };
     });

package/lib/{routes → v2}/yystv/docs.js

@@ -1,6 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const date = require('@/utils/date');
+const { parseRelativeDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const url = `https://www.yystv.cn/docs`;
@@ -18,7 +18,7 @@ module.exports = async (ctx) => {
             const info = {
                 title: $('.list-article-title', this).text(),
                 link: 'https://www.yystv.cn' + $('a', this).attr('href'),
-                pubDate: date($('.c-999', this).text()),
+                pubDate: parseRelativeDate($('.c-999', this).text()),
                 author: $('.handler-author-link', this).text(),
                 description: $('.list-article-intro', this).text(),
             };

package/lib/v2/yystv/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/category/:category': ['LightStrawberry'],
+    '/docs': ['HaitianLiu'],
+};

package/lib/v2/yystv/radar.js

@@ -0,0 +1,49 @@
+module.exports = {
+    'yystv.cn': {
+        _name: '游研社',
+        '.': [
+            {
+                title: '推游',
+                docs: 'https://docs.rsshub.app/game.html#you-yan-she',
+                source: '/b/recommend',
+                target: '/yystv/category/recommend',
+            },
+            {
+                title: '游戏史',
+                docs: 'https://docs.rsshub.app/game.html#you-yan-she',
+                source: '/b/history',
+                target: '/yystv/category/history',
+            },
+            {
+                title: '大事件',
+                docs: 'https://docs.rsshub.app/game.html#you-yan-she',
+                source: '/b/big',
+                target: '/yystv/category/big',
+            },
+            {
+                title: '文化',
+                docs: 'https://docs.rsshub.app/game.html#you-yan-she',
+                source: '/b/culture',
+                target: '/yystv/category/culture',
+            },
+            {
+                title: '趣闻',
+                docs: 'https://docs.rsshub.app/game.html#you-yan-she',
+                source: '/b/news',
+                target: '/yystv/category/news',
+            },
+            {
+                title: '经典回顾',
+                docs: 'https://docs.rsshub.app/game.html#you-yan-she',
+                source: '/b/retro',
+                target: '/yystv/category/retro',
+            },
+            {
+                title: '全部文章',
+                docs: 'https://docs.rsshub.app/game.html#you-yan-she',
+                source: '/docs',
+                target: '/yystv/docs',
+            },
+        ],
+    },
+};

package/lib/v2/yystv/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/category/:category', require('./category'));
+    router.get('/docs', require('./docs'));
+};