rsshub 1.0.0-master.f2ed98e → 1.0.0-master.f2eee10

package/lib/v2/youtube/charts.js

@@ -0,0 +1,97 @@
+const got = require('@/utils/got');
+const { renderDescription } = require('./utils');
+const config = require('@/config').value;
+
+module.exports = async (ctx) => {
+    const contentMap = {
+        TopArtists: {
+            contentKey: 'artists',
+            viewKey: 'artistViews',
+            title: 'Top artists',
+        },
+        TopSongs: {
+            contentKey: 'trackTypes',
+            viewKey: 'trackViews',
+            title: 'Top songs',
+        },
+        TopVideos: {
+            contentKey: 'videos',
+            viewKey: 'videoViews',
+            title: 'Top music videos',
+        },
+        TrendingVideos: {
+            contentKey: 'videos',
+            viewKey: 'videoViews',
+            title: 'Trending',
+        },
+    };
+
+    const { category = 'TopVideos', country } = ctx.params;
+    const embed = !ctx.params.embed;
+
+    const { content } = await ctx.cache.tryGet(
+        `youtube:charts:${country ? country : 'global'}`,
+        async () => {
+            const { data } = await got.post('https://charts.youtube.com/youtubei/v1/browse', {
+                searchParams: {
+                    alt: 'json',
+                    key: 'AIzaSyCzEW7JUJdSql0-2V4tHUb6laYm4iAE_dM',
+                },
+                json: {
+                    browseId: 'FEmusic_analytics_charts_home',
+                    context: {
+                        capabilities: {},
+                        client: {
+                            clientName: 'WEB_MUSIC_ANALYTICS',
+                            clientVersion: '0.2',
+                            experimentIds: [],
+                            experimentsToken: '',
+                            gl: 'US',
+                            hl: 'en',
+                            theme: 'MUSIC',
+                        },
+                        request: {
+                            internalExperimentFlags: [],
+                        },
+                    },
+                    query: `chart_params_type=WEEK&perspective=CHART&flags=viral_video_chart&selected_chart=TRACKS&chart_params_id=weekly:0:0${country ? `:${country}` : ''}`,
+                },
+            });
+            return data.contents.sectionListRenderer.contents[0].musicAnalyticsSectionRenderer;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const { entityId } = content.perspectiveMetadata;
+    let items;
+
+    if (category === 'TopArtists') {
+        items = content[contentMap[category].contentKey][0][contentMap[category].viewKey].map((item) => ({
+            title: item.name,
+            link: `https://charts.youtube.com/artist/${encodeURIComponent(item.id)}`,
+            guid: `youtube:charts:${category}:${entityId}:${item.id}`,
+        }));
+    } else {
+        items = content[contentMap[category].contentKey][category !== 'TrendingVideos' ? 0 : 1][contentMap[category].viewKey].map((item) => {
+            const videoId = category === 'TopSongs' ? item.encryptedVideoId : item.id;
+            const author = item.artists
+                .filter((a) => a.name)
+                .map((artist) => artist.name)
+                .join(', ');
+            return {
+                title: `${item.title ?? item.name} - ${author}`,
+                description: renderDescription(embed, videoId, item.thumbnail?.thumbnails.pop().url, ''),
+                link: `https://www.youtube.com/watch?v=${videoId}`,
+                guid: `youtube:charts:${category}:${entityId}:${item.id}`,
+                author,
+            };
+        });
+    }
+
+    ctx.state.data = {
+        title: `YouTube Music Charts - ${contentMap[category].title}`,
+        link: `https://charts.youtube.com/charts/${category}/${country ? country : 'global'}`,
+        item: items,
+    };
+};

package/lib/v2/youtube/community.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseRelativeDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const { isYouTubeChannelId } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { handle } = ctx.params;
+
+    let urlPath = handle;
+    if (isYouTubeChannelId(handle)) {
+        urlPath = `channel/${handle}`;
+    }
+
+    const { data: response } = await got(`https://www.youtube.com/${urlPath}/community`);
+    const $ = cheerio.load(response);
+    const ytInitialData = JSON.parse(
+        $('script')
+            .text()
+            .match(/ytInitialData = ({.*?});/)[1]
+    );
+
+    const channelMetadata = ytInitialData.metadata.channelMetadataRenderer;
+    const username = channelMetadata.title;
+    const communityTab = ytInitialData.contents.twoColumnBrowseResultsRenderer.tabs.find((tab) => tab.tabRenderer.endpoint.commandMetadata.webCommandMetadata.url.endsWith('/community'));
+    const list = communityTab.tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents;
+
+    if (list[0].messageRenderer) {
+        throw Error(list[0].messageRenderer.text.runs[0].text);
+    }
+
+    const items = list
+        .filter((i) => i.backstagePostThreadRenderer)
+        .map((item) => {
+            const post = item.backstagePostThreadRenderer.post.backstagePostRenderer;
+            const media = post.backstageAttachment?.postMultiImageRenderer?.images.map((i) => i.backstageImageRenderer.image.thumbnails.pop()) ?? [post.backstageAttachment?.backstageImageRenderer?.image.thumbnails.pop()];
+            return {
+                title: post.contentText.runs[0].text,
+                description: art(path.join(__dirname, 'templates', 'community.art'), {
+                    runs: post.contentText.runs,
+                    media,
+                }),
+                link: `https://www.youtube.com/post/${post.postId}`,
+                author: post.authorText.runs[0].text,
+                pubDate: parseRelativeDate(post.publishedTimeText.runs[0].text.split('(')[0]),
+            };
+        });
+
+    ctx.state.data = {
+        title: `${username} - Community - YouTube`,
+        link: channelMetadata.channelUrl,
+        description: channelMetadata.description,
+        item: items,
+    };
+};

package/lib/v2/youtube/maintainer.js

@@ -1,5 +1,8 @@
 module.exports = {
+    '/c/:id/:embed?': ['TonyRL'],
     '/channel/:id/:embed?': ['DIYgod'],
+    '/charts/:category?/:country?/:embed?': ['TonyRL'],
+    '/community/:handle': ['TonyRL'],
     '/playlist/:id/:embed?': ['HenryQW'],
     '/subscriptions/:embed?': ['TonyRL'],
     '/user/:username/:embed?': ['DIYgod'],

package/lib/v2/youtube/radar.js

@@ -1,6 +1,14 @@
 module.exports = {
     'youtube.com': {
         _name: 'YouTube',
+        charts: [
+            {
+                title: '音乐排行榜',
+                docs: 'https://docs.rsshub.app/social-media.html#youtube',
+                source: ['/charts/:category/:country/*', '/charts/:category/:country', '/charts/:category'],
+                target: (params) => `/youtube/charts/${params.category}${params.country ? params.country : ''}`,
+            },
+        ],
         www: [
             {
                 title: '用户',
@@ -8,6 +16,12 @@ module.exports = {
                 source: '/user/:username',
                 target: '/youtube/user/:username',
             },
+            {
+                title: '用户',
+                docs: 'https://docs.rsshub.app/social-media.html#youtube',
+                source: '/:handle',
+                target: (params) => (params.handle.startsWith('@') ? `/youtube/user/${params.handle}` : ''),
+            },
             {
                 title: '频道',
                 docs: 'https://docs.rsshub.app/social-media.html#youtube',
@@ -20,6 +34,12 @@ module.exports = {
                 source: '/c/:id',
                 target: '/youtube/c/:id',
             },
+            {
+                title: '社群',
+                docs: 'https://docs.rsshub.app/social-media.html#youtube',
+                source: ['/channel/:handle/community', '/channel/:handle', '/:handle/community', '/:handle/featured', '/:handle'],
+                target: (params) => (params.handle.startsWith('@') || params.handle.startsWith('UC') ? `/youtube/community/${params.handle}` : ''),
+            },
             {
                 title: '播放列表',
                 docs: 'https://docs.rsshub.app/social-media.html#youtube',

package/lib/v2/youtube/router.js

@@ -1,6 +1,8 @@
 module.exports = (router) => {
     router.get('/c/:username/:embed?', require('./custom'));
     router.get('/channel/:id/:embed?', require('./channel'));
+    router.get('/charts/:category?/:country?/:embed?', require('./charts'));
+    router.get('/community/:handle', require('./community'));
     router.get('/playlist/:id/:embed?', require('./playlist'));
     router.get('/subscriptions/:embed?', require('./subscriptions'));
     router.get('/user/:username/:embed?', require('./user'));

package/lib/v2/youtube/templates/community.art

@@ -0,0 +1,17 @@
+{{ if runs }}
+    {{ each runs run }}
+        {{ if run.navigationEndpoint }}
+            {{ set url = run.navigationEndpoint.commandMetadata.webCommandMetadata.url }}
+            <a href="{{ url.startsWith('https://') ? url : `https://www.youtube.com${url}` }}">{{ run.text }}</a>
+        {{ else }}
+            {{@ run.text.replace(/\n/g, '<br>') }}
+        {{ /if }}
+    {{ /each }}
+{{ /if }}
+
+{{ if media }}
+<br>
+    {{ each media i }}
+        {{ if i?.url }}<img src="{{ i.url }}">{{ /if }}
+    {{ /each }}
+{{ /if }}

package/lib/v2/youtube/user.js

@@ -6,7 +6,7 @@ const cheerio = require('cheerio');
 
 module.exports = async (ctx) => {
     if (!config.youtube || !config.youtube.key) {
-        throw 'YouTube RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>';
+        throw Error('YouTube RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
     }
     const username = ctx.params.username;
     const embed = !ctx.params.embed;
@@ -17,7 +17,7 @@ module.exports = async (ctx) => {
         const link = `https://www.youtube.com/${username}`;
         const response = await got(link);
         const $ = cheerio.load(response.data);
-        const channelId = $('meta[itemprop="channelId"]').attr('content');
+        const channelId = $('meta[itemprop="identifier"]').attr('content');
         channelName = $('meta[itemprop="name"]').attr('content');
         playlistId = (await utils.getChannelWithId(channelId, 'contentDetails', ctx.cache)).data.items[0].contentDetails.relatedPlaylists.uploads;
     }

package/lib/v2/youtube/utils.js

@@ -115,7 +115,7 @@ const youtubeUtils = {
         if (!accessToken) {
             const data = await youtubeOAuth2Client.getAccessToken();
             accessToken = data.token;
-            await cache.set('youtube:accessToken', accessToken, ~~((data.res.data.expiry_date - Date.now()) / 1000)); // ~3600s
+            await cache.set('youtube:accessToken', accessToken, 3600); // ~3600s
         }
         youtubeOAuth2Client.setCredentials({ access_token: accessToken, refresh_token: config.youtube.refreshToken });
 
@@ -136,6 +136,8 @@ const youtubeUtils = {
         }
         return res;
     },
+    // taken from https://webapps.stackexchange.com/a/101153
+    isYouTubeChannelId: (id) => /^UC[\w-]{21}[AQgw]$/.test(id),
 };
 
 module.exports = youtubeUtils;

package/lib/v2/yuque/book.js

@@ -60,6 +60,20 @@ module.exports = async (ctx) => {
                         elem.remove();
                     }
                 });
+                // obtain real video src
+                for await (const v of $('video').toArray()) {
+                    const $v = $(v);
+                    const src = $v.attr('src');
+                    if (src.startsWith('inputs')) {
+                        const { data } = await got(`${baseUrl}/api/video`, {
+                            searchParams: {
+                                video_id: src,
+                            },
+                        });
+                        const { info } = data.data;
+                        $v.replaceWith(`<video controls preload='none' poster='${info.cover}'><source src='${info.url}' type='video/mp4'></video>`);
+                    }
+                }
 
                 item.description = $.html();
                 item.author = data.contributors.map((c) => c.name).join(', ');

package/lib/v2/yuque/utils.js

@@ -35,6 +35,10 @@ const card2Html = (elem, link) => {
         case 'table':
             html = value.html;
             break;
+        case 'video':
+            // fake video src
+            html = `<video src='${value.videoId}'></video>`;
+            break;
         default:
             throw Error(`Unhandled card on ${link}: ${name}`);
     }

package/lib/v2/zagg/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/new-arrivals/:query?': ['NavePnow'],
+};

package/lib/v2/zagg/new-arrivals.js

@@ -0,0 +1,52 @@
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+const cheerio = require('cheerio');
+const host = 'https://www.zagg.com/en_us';
+module.exports = async (ctx) => {
+    const query = ctx.params.query;
+    const params = new URLSearchParams(query);
+    const brands = params.get('brand');
+    const categories = params.get('cat');
+
+    const url = `${host}/new-arrivals`;
+    const response = await got({
+        headers: {
+            'X-Requested-With': 'XMLHttpRequest',
+        },
+        method: 'post',
+        url,
+        searchParams: {
+            cat: categories,
+            brand: brands,
+        },
+    });
+    const products = response.data.products;
+
+    const $ = cheerio.load(products);
+    const list = $('.item.product.product-item')
+        .map(function () {
+            const data = {};
+            const details = $(this).find('.product.details-box').html();
+            data.link = $(this).find('.product-item-link').eq(0).attr('href');
+            data.title = $(this).find('.product-item-link').text();
+            const regex = /(https.*?)\?/;
+            const imgUrl = $(this).find('img').eq(0).attr('data-src').match(regex)[1];
+            const img = art(path.join(__dirname, 'templates/new-arrivals.art'), {
+                imgUrl,
+            });
+            data.description = details + img;
+            return data;
+        })
+        .get();
+    ctx.state.data = {
+        title: 'Zagg - New Arrivals',
+        link: response.url,
+        description: 'Zagg - New Arrivals',
+        item: list.map((item) => ({
+            title: item.title,
+            description: item.description,
+            link: item.link,
+        })),
+    };
+};

package/lib/v2/zagg/radar.js

@@ -0,0 +1,16 @@
+module.exports = {
+    'zagg.com': {
+        _name: 'New Arrivals',
+        '.': [
+            {
+                title: 'Zagg - New Arrivals',
+                docs: 'https://docs.rsshub.app/shopping.html#zagg',
+                source: ['/en_us/new-arrivals'],
+                target: (_, url) => {
+                    const queryString = url.split('?')[1];
+                    return `/zagg/new-arrivals/${queryString}`;
+                },
+            },
+        ],
+    },
+};

package/lib/v2/zagg/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/new-arrivals/:query?', require('./new-arrivals'));
+};

package/lib/v2/zagg/templates/new-arrivals.art

@@ -0,0 +1,3 @@
+<div>
+    <img src={{imgUrl}}>
+</div>

package/lib/v2/zaobao/util.js

@@ -4,6 +4,7 @@ const date = require('@/utils/date');
 const timezone = require('@/utils/timezone');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { base32 } = require('rfc4648');
 
 const baseUrl = 'https://www.zaobao.com';
 const got_ins = got.extend({
@@ -35,22 +36,14 @@ const parseList = async (ctx, sectionUrl) => {
         data = $('.clearfix').find('.list-block');
     }
 
-    const title = $('#breadcrumbs > a')
-        .toArray()
-        .reduce((acc, cV, cI) => {
-            if (cI > 0) {
-                return acc + '-' + $(cV).text();
-            }
-            return '';
-        }, '');
+    const title = $('meta[property="og:title"]').attr('content');
+
     const resultList = await Promise.all(
-        data.toArray().map(async (item) => {
+        data.toArray().map((item) => {
             const $item = $(item);
             // addBack: for HK version
             let link = $item.find('a').addBack('a')[0].attribs.href;
 
-            let resultItem = {};
-
             if (link[0] !== '/') {
                 // https://www.zaobao.com/interactive-graphics
                 const title = $item.find('a').text();
@@ -71,31 +64,29 @@ const parseList = async (ctx, sectionUrl) => {
                     pubDate = pubDate === dateString ? undefined : timezone(pubDate, +8);
                 }
 
-                resultItem = {
+                return {
                     title,
                     description: title,
                     pubDate,
                     link,
                 };
-                return Promise.resolve(resultItem);
-            } else {
-                link = baseUrl + link;
             }
 
-            const value = await ctx.cache.get(link);
+            link = baseUrl + link;
 
-            if (value) {
-                resultItem = JSON.parse(value);
-            } else {
+            return ctx.cache.tryGet(link, async () => {
                 const article = await got_ins.get(link);
                 const $1 = cheerio.load(article.data);
 
-                let time = $1('.date-published', '.content').slice(0, 1).text().trim();
-                if (!time) {
-                    // for HK version
-                    time = $1('.author-info').find('.mgt10').slice(0, 1).text().trim();
-                }
-                time = time.replace('年', '-').replace('月', '-').replace('日', '').replace('发布 /', '');
+                const time = (() => {
+                    if ($1("head script[type='application/json']").text() !== '') {
+                        // SG
+                        return new Date(Number(JSON.parse($1("head script[type='application/json']").text())?.articleDetails?.created) * 1000);
+                    } else {
+                        // HK
+                        return new Date(JSON.parse($1("head script[type='application/ld+json']").eq(1).text())?.datePublished);
+                    }
+                })();
 
                 $1('.overlay-microtransaction').remove();
                 $1('#video-freemium-player').remove();
@@ -104,6 +95,7 @@ const parseList = async (ctx, sectionUrl) => {
                 let articleBodyNode = $1('.article-content-rawhtml');
                 if (articleBodyNode.length === 0) {
                     // for HK version
+                    orderContent($1('.article-body'));
                     articleBodyNode = $1('.article-body');
                 }
 
@@ -152,20 +144,17 @@ const parseList = async (ctx, sectionUrl) => {
                     });
                 }
 
-                resultItem = {
+                return {
                     // <- for SG version  -> for HK version
                     title: $1('h1', '.content').text().trim() || $1('h1.article-title').text(),
                     description: art(path.join(__dirname, 'templates/zaobao.art'), {
                         articleBody,
                         imageDataArray,
                     }),
-                    pubDate: timezone(time, +8), // zaobao seems always UTC+8
+                    pubDate: time,
                     link,
                 };
-                ctx.cache.set(link, JSON.stringify(resultItem));
-            }
-
-            return Promise.resolve(resultItem);
+            });
         })
     );
     return {
@@ -173,6 +162,24 @@ const parseList = async (ctx, sectionUrl) => {
         resultList,
     };
 };
+
+const orderContent = (parent) => {
+    parent
+        .children()
+        .toArray()
+        .sort((a, b) => {
+            const index = Buffer.from(base32.parse('GM======')).toString(); // substring(3)
+            a = Buffer.from(base32.parse(parent.find(a).data('s').substring(index))).toString();
+            b = Buffer.from(base32.parse(parent.find(b).data('s').substring(index))).toString();
+            return a - b;
+        })
+        .forEach((e, i) => {
+            parent.find(e).attr('s', i);
+            parent.append(e);
+        });
+};
+
 module.exports = {
     parseList,
+    orderContent,
 };

package/lib/v2/zcool/radar.js

@@ -11,37 +11,37 @@ module.exports = {
             {
                 title: '发现 - 精选 - 全部推荐',
                 docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
+                source: '/',
                 target: '/zcool/discover/all',
             },
             {
                 title: '发现 - 精选 - 首页推荐',
                 docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
+                source: '/',
                 target: '/zcool/discover/home',
             },
             {
                 title: '发现 - 精选 - 编辑精选',
                 docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
+                source: '/',
                 target: '/zcool/discover/home',
             },
             {
                 title: '发现 - 精选 - 文章 - 编辑精选',
                 docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
+                source: '/',
                 target: '/zcool/discover/article',
             },
             {
                 title: '作品榜单',
                 docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
+                source: '/',
                 target: '/zcool/top/design',
             },
             {
                 title: '文章榜单',
                 docs: 'https://docs.rsshub.app/design.html#zhan-ku',
-                source: '',
+                source: '/',
                 target: '/zcool/top/article',
             },
             {

package/lib/v2/zhihu/answers.js

@@ -19,16 +19,20 @@ module.exports = async (ctx) => {
     let name = data[0].author.name;
 
     if (name === '知乎用户') {
-        const info = await got({
-            method: 'get',
-            url: `https://www.zhihu.com/api/v4/members/${id}/activities?limit=1`,
-            headers: {
-                ...utils.header,
-                Referer: `https://www.zhihu.com/people/${id}/activities`,
-                Authorization: 'oauth c3cef7c66a1843f8b3a9e6a1e3160e20', // hard-coded in js
-            },
+        const userProfile = await ctx.cache.tryGet(`zhihu:profile:${id}`, async () => {
+            const apiPath = `/api/v4/profile/${id}/infinity`;
+
+            const { data } = await got({
+                method: 'get',
+                url: `https://www.zhihu.com${apiPath}`,
+                headers: {
+                    ...utils.header,
+                    Referer: `https://www.zhihu.com/people/${id}`,
+                },
+            });
+            return data.data;
         });
-        name = info.data.data[0].actor.name;
+        name = userProfile.name;
     }
 
     const items = await Promise.all(

package/lib/v2/zhihu/radar.js

@@ -93,13 +93,13 @@ module.exports = {
             {
                 title: '日报',
                 docs: 'https://docs.rsshub.app/social-media.html#zhi-hu',
-                source: '',
+                source: '/',
                 target: '/zhihu/daily',
             },
             {
                 title: '日报',
                 docs: 'https://docs.rsshub.app/social-media.html#zhi-hu',
-                source: '/*tpath',
+                source: '/*',
                 target: '/zhihu/daily',
             },
         ],

package/lib/v2/zhubai/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
+    '/top20': ['nczitzk'],
     '/:id': ['naixy28'],
 };

package/lib/v2/zhubai/radar.js

@@ -8,5 +8,12 @@ module.exports = {
                 source: ['/'],
             },
         ],
+        analy: [
+            {
+                title: 'TOP 20',
+                docs: 'https://docs.rsshub.app/blog.html#zhu-bai',
+                source: ['/'],
+            },
+        ],
     },
 };

package/lib/v2/zhubai/router.js

@@ -1,3 +1,4 @@
 module.exports = function (router) {
+    router.get('/top20', require('./top20'));
     router.get('/:id', require('./index'));
 };