rsshub 1.0.0-master.f992838 → 1.0.0-master.f9a482d

package/lib/v2/xsijishe/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/forum/:fid': ['akynazh'],
+};

package/lib/v2/xsijishe/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'xsijishe.com': {
+        _name: '司机社',
+        '.': [
+            {
+                title: '论坛',
+                docs: 'https://docs.rsshub.app/bbs.html#si-ji-she',
+                source: ['/*'],
+                target: (_, url) => {
+                    const re = /forum-(\d+)-/;
+                    const res = re.exec(url);
+                    if (res) {
+                        return `/xsijishe/forum/${res[1]}`;
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/xsijishe/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/forum/:fid', require('./forum'));
+};

package/lib/v2/xueqiu/user.js

@@ -2,6 +2,8 @@ const got = require('@/utils/got');
 const queryString = require('query-string');
 const { parseDate } = require('@/utils/parse-date');
 
+const rootUrl = 'https://xueqiu.com';
+
 module.exports = async (ctx) => {
     const id = ctx.params.id;
     const type = ctx.params.type || 10;
@@ -17,13 +19,13 @@ module.exports = async (ctx) => {
 
     const res1 = await got({
         method: 'get',
-        url: 'https://xueqiu.com/',
+        url: rootUrl,
     });
     const token = res1.headers['set-cookie'].find((s) => s.startsWith('xq_a_token=')).split(';')[0];
 
     const res2 = await got({
         method: 'get',
-        url: 'https://xueqiu.com/v4/statuses/user_timeline.json',
+        url: `${rootUrl}/v4/statuses/user_timeline.json`,
         searchParams: queryString.stringify({
             user_id: id,
             type,
@@ -31,24 +33,43 @@ module.exports = async (ctx) => {
         }),
         headers: {
             Cookie: token,
-            Referer: `https://xueqiu.com/u/${id}`,
+            Referer: `${rootUrl}/u/${id}`,
         },
     });
     const data = res2.data.statuses.filter((s) => s.mark !== 1); // 去除置顶动态
 
+    const items = await Promise.all(
+        data.map((item) =>
+            ctx.cache.tryGet(item.target, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: rootUrl + item.target,
+                    headers: {
+                        Referer: `${rootUrl}/u/${id}`,
+                        Cookie: token,
+                    },
+                });
+
+                const data = JSON.parse(detailResponse.data.match(/SNOWMAN_STATUS = (.*?});/)[1]);
+                item.text = data.text;
+
+                const retweetedStatus = item.retweeted_status ? `<blockquote>${item.retweeted_status.user.screen_name}:&nbsp;${item.retweeted_status.description}</blockquote>` : '';
+                const description = item.description + retweetedStatus;
+
+                return {
+                    title: item.title ? item.title : description.replace(/<[^>]+>/g, ''),
+                    description: item.text ? item.text + retweetedStatus : description,
+                    pubDate: parseDate(item.created_at),
+                    link: rootUrl + item.target,
+                };
+            })
+        )
+    );
+
     ctx.state.data = {
         title: `${data[0].user.screen_name} 的雪球${typename[type]}动态`,
-        link: `https://xueqiu.com/u/${id}`,
+        link: `${rootUrl}/u/${id}`,
         description: `${data[0].user.screen_name} 的雪球${typename[type]}动态`,
-        item: data.map((item) => {
-            const retweetedStatus = item.retweeted_status ? `<blockquote>${item.retweeted_status.user.screen_name}:&nbsp;${item.retweeted_status.description}</blockquote>` : '';
-            const description = item.description + retweetedStatus;
-            return {
-                title: item.title ? item.title : description.replace(/<[^>]+>/g, ''),
-                description: item.text ? item.text + retweetedStatus : description,
-                pubDate: parseDate(item.created_at),
-                link: `https://xueqiu.com${item.target}`,
-            };
-        }),
+        item: items,
     };
 };

package/lib/v2/xyzrank/radar.js

@@ -5,26 +5,26 @@ module.exports = {
             {
                 title: '热门节目',
                 docs: 'https://docs.rsshub.app/multimedia.html#zhong-wen-bo-ke-bang-re-men-jie-mu',
-                source: ['/#/', '/'],
+                source: ['/'],
                 target: '/xyzrank',
             },
             {
                 title: '热门播客',
                 docs: 'https://docs.rsshub.app/multimedia.html#zhong-wen-bo-ke-bang-re-men-bo-ke',
-                source: ['/#/hot-podcasts', '/'],
-                target: '/xyzrank/hot-podcasts',
+                source: ['/'],
+                target: (_, url) => (new URL(url).hash === '#/hot-podcasts' ? '/xyzrank/hot-podcasts' : null),
             },
             {
                 title: '新锐节目',
                 docs: 'https://docs.rsshub.app/multimedia.html#zhong-wen-bo-ke-bang-xin-rui-jie-mu',
-                source: ['/#/hot-episodes-new', '/'],
-                target: '/xyzrank/hot-episodes-new',
+                source: ['/'],
+                target: (_, url) => (new URL(url).hash === '#/hot-episodes-new' ? '/xyzrank/hot-episodes-new' : null),
             },
             {
                 title: '新锐播客',
                 docs: 'https://docs.rsshub.app/multimedia.html#zhong-wen-bo-ke-bang-xin-rui-bo-ke',
-                source: ['/#/new-podcasts', '/'],
-                target: '/xyzrank/new-podcasts',
+                source: ['/'],
+                target: (_, url) => (new URL(url).hash === '#/new-podcasts' ? '/xyzrank/new-podcasts' : null),
             },
         ],
     },

package/lib/v2/yahoo/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:region/:category?': ['KeiLongW'],
+};

package/lib/v2/yahoo/news/index.js

@@ -0,0 +1,46 @@
+const got = require('@/utils/got');
+const parser = require('@/utils/rss-parser');
+const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
+
+module.exports = async (ctx) => {
+    const region = ctx.params.region === 'en' ? '' : ctx.params.region.toLowerCase();
+    if (!isValidHost(region) && region !== '') {
+        throw Error('Invalid region');
+    }
+    const category = ctx.params.category ? ctx.params.category.toLowerCase() : '';
+    const rssUrl = `https://${region ? region + '.' : ''}news.yahoo.com/rss/${category}`;
+    const feed = await parser.parseURL(rssUrl);
+    const title = feed.title;
+    const link = feed.link;
+    const description = feed.description;
+    const filteredItems = feed.items.filter((item) => !item.link.includes('promotions') && new URL(item.link).hostname.match(/.*\.yahoo\.com$/));
+    const items = await Promise.all(
+        filteredItems.map(async (item) => {
+            const description = await ctx.cache.tryGet(item.link, async () => {
+                const response = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const $ = cheerio.load(response.data);
+                $('.caas-content-byline-wrapper, .caas-xray-wrapper, .caas-header, .caas-readmore').remove();
+                return $('.caas-content-wrapper').html();
+            });
+            const single = {
+                title: item.title,
+                description,
+                pubDate: item.pubDate,
+                link: item.link,
+            };
+            return single;
+        })
+    );
+
+    ctx.state.data = {
+        title,
+        link,
+        description,
+        item: items,
+    };
+};

package/lib/v2/yahoo/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'yahoo.com': {
+        _name: 'Yahoo',
+        '.': [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yahoo',
+                source: ['/'],
+                target: '/yahoo/news/:region/:category?',
+            },
+        ],
+    },
+};

package/lib/v2/yahoo/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/news/:region/:category?', require('./news/index'));
+};

package/lib/v2/ycwb/index.js

@@ -0,0 +1,82 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const node = ctx.params.node ?? 1;
+    const currentUrl = `https://6api.ycwb.com/app_if/jy/getArticles?nodeid=${node}&pagesize=15`;
+
+    const { data: response } = await got(currentUrl);
+
+    const list = response.artiles.map((item) => ({
+        title: item.TITLE,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            thumb: item.PICLINKS,
+            description: item.ABSTRACT,
+        }),
+        pubDate: timezone(parseDate(item.PUBTIME), +8),
+        link: item.PUBURL,
+        nodeName: item.NODENAME,
+    }));
+
+    let nodeName = '';
+    let nodeLink = '';
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                const $comments = content('.main_article')
+                    .contents()
+                    .filter(function () {
+                        return this.nodeType === 8;
+                    });
+                $comments.each(function () {
+                    // Remove useless comments
+                    if (this.data.match(/audioPlayer|audio-box/)) {
+                        this.data = '';
+                    }
+                    // Filter author from comments
+                    if (this.data.match(/author/)) {
+                        item.author = this.data.split('<author>')[1].split('</author>')[0];
+                    }
+                });
+
+                nodeName = nodeName === '' ? item.nodeName : nodeName;
+
+                // Filter node link from content('.path a')
+                const children = content('.path').children('a');
+                for (const child of children) {
+                    if (content(child).text() === nodeName && nodeLink === '') {
+                        nodeLink = content(child).attr('href');
+                    }
+                }
+
+                content('span').removeAttr('style').removeAttr('class');
+                content('img').removeAttr('style').removeAttr('class').removeAttr('placement').removeAttr('data-toggle').removeAttr('trigger').removeAttr('referrerpolicy');
+                content('br').removeAttr('style').removeAttr('class');
+                content('p').removeAttr('style').removeAttr('class');
+                content('.space10, .ddf').remove();
+
+                item.description += content('.main_article').html() ?? '';
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `羊城晚报金羊网 - ${nodeName}`,
+        link: String(nodeLink === '' ? 'https://www.ycwb.com/' : nodeLink),
+        item: items,
+    };
+};

package/lib/v2/ycwb/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:node': ['TimWu007'],
+};

package/lib/v2/ycwb/radar.js

@@ -0,0 +1,12 @@
+module.exports = {
+    'xwlb.com.cn': {
+        _name: '羊城晚报金羊网',
+        '.': [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yang-cheng-wan-bao-jin-yang-wang',
+                source: ['/'],
+            },
+        ],
+    },
+};

package/lib/v2/ycwb/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:node', require('./index'));
+};

package/lib/v2/ycwb/templates/description.art

@@ -0,0 +1,7 @@
+{{ if thumb }}
+<img src="{{ thumb }}"><br>
+{{ /if }}
+{{ if description }}
+    <blockquote><p>{{ description }}</p></blockquote>
+    <br>
+{{ /if }}

package/lib/v2/yicai/utils.js

@@ -28,7 +28,7 @@ module.exports = {
             }),
         }));
 
-        return await Promise.all(
+        return Promise.all(
             items.map((item) =>
                 tryGet(item.link, async () => {
                     const detailResponse = await got({

package/lib/v2/yomiuri/maintainer.js

@@ -1,3 +1,3 @@
 module.exports = {
-    '/yomiuri/:category?': ['Arracc'],
+    '/:category?': ['Arracc'],
 };

package/lib/v2/yomiuri/news.js

@@ -4,126 +4,76 @@ const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const category = ctx.params.category;
+    const { category = 'news' } = ctx.params;
     const url = `https://www.yomiuri.co.jp/${category}`;
 
-    const response = await got({
-        method: 'get',
-        url,
-    });
+    const response = await got(url);
     const data = response.data;
     const $ = cheerio.load(data);
 
-    let categoryName;
-    let list = null;
+    let list;
     if (category === 'news') {
-        categoryName = '新着・速報';
-
-        list = $('div#latest_list_news').find('article').append($('div#latest_list_news_more').find('article'));
-
-        list = list.map((index, item) => {
-            item = $(item);
-            // remove matome title
-            const matomeTitle = item.find('div.c-matome-title');
-            if (matomeTitle.length !== 0) {
-                matomeTitle.remove();
-            }
-            // remove member only
-            const lockedIcon = item.find('svg[class=icon-locked]');
-            if (lockedIcon.length === 0) {
-                return item;
-            } else {
-                return null;
-            }
-        });
+        list = $('.news-top-latest__list .news-top-latest__list-item__inner')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+                const a = item.find('h3 a');
+                return {
+                    title: a.text(),
+                    link: a.attr('href'),
+                    pubDate: timezone(parseDate(item.find('time').attr('datetime')), +9),
+                    locked: item.find('.icon-locked').length,
+                };
+            });
     } else {
-        let parentSelector = 'div.p-category-organization,.p-category-time-series';
-        // igoshougi channel sepcial div selector
-        if (category === 'igoshougi') {
-            parentSelector = 'div.uni-news-igoshougi-top';
-        }
-        categoryName = $('h1.p-header-category-current-title,h1.p-header-category-content-title').text();
-        // p-header-category-content-title
-        list = $(parentSelector).find('li').not('.p-member-only').not('.p-ad-list-item');
-        list = list.map((index, item) => {
-            item = $(item);
-            // remove matome title
-            const matomeTitle = item.find('div.c-matome-title');
-            if (matomeTitle.length !== 0) {
-                matomeTitle.remove();
-            }
-            return item;
-        });
-    }
-
-    // format category name
-    let formattedCategoryName = categoryName;
-    if (categoryName !== '') {
-        formattedCategoryName = ' - ' + categoryName.trim();
+        $('.p-category-reading-recommend').remove();
+        list = $('.layout-contents__main .c-list-title')
+            .toArray()
+            .map((item) => {
+                item = $(item);
+                const a = item.find('h3 a');
+                const parent = item.parent();
+                return {
+                    title: a.text(),
+                    link: a.attr('href'),
+                    pubDate: timezone(parseDate(parent.find('time').attr('datetime')), +9),
+                    locked: parent.find('.c-list-member-only').length,
+                };
+            });
     }
 
     const items = await Promise.all(
-        list
-            .map(async (index, item) => {
-                item = $(item);
-
-                let link = '';
-                if (categoryName === '新着・速報') {
-                    link = item.find('h3').children('a').attr('href');
-                } else {
-                    link = item.find('a').attr('href');
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (item.locked) {
+                    return item;
                 }
 
-                const description = await ctx.cache.tryGet(link, async () => {
-                    const response = await got(link);
-                    const $ = cheerio.load(response.data);
-                    const mainContent = $('div.p-main-contents');
-                    // remove articles recomended
-                    const recommendNode = mainContent.find('aside.ev-article-manual-related__article-inline');
-                    if (recommendNode.length !== 0) {
-                        recommendNode.remove();
-                    }
-                    return mainContent.html();
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                const mainContent = $('.p-main-contents');
+
+                mainContent.find('[class^=ev-article], svg').remove();
+                mainContent.find('img').each((_, img) => {
+                    img.attribs.src = img.attribs.src.split('?')[0];
                 });
 
-                let tag = '';
-                if (categoryName === '新着・速報') {
-                    const tag_table = {
-                        news: '新着・速報',
-                        national: '社会',
-                        politics: '政治',
-                        economy: '経済',
-                        sports: 'スポーツ',
-                        world: '国際',
-                        science: '科学・ＩＴ',
-                        election: '選挙・世論調査',
-                        culture: 'エンタメ・文化',
-                        igoshougi: '囲碁・将棋',
-                        life: 'ライフ',
-                        local: '地域',
-                        editorial: '社説',
-                        medical: '医療・健康',
-                        olympic: 'オリンピック',
-                        koushitsu: '皇室',
-                    };
-                    const tag_value = tag_table[link.split('/')[3]];
-                    tag = tag_value ? tag_value : 'その他';
-                    tag = '[' + tag + '] ';
-                }
+                item.description = mainContent.html();
+                item.pubDate = parseDate($('meta[property="article:published_time"]').attr('content')); // 2023-05-17T22:33:00+09:00
+                item.updated = parseDate($('meta[property="article:modified_time"]').attr('content'));
 
-                return {
-                    title: tag + item.find('a').text(),
-                    description,
-                    pubDate: timezone(parseDate(item.find('time').prop('datetime')), +9),
-                    link,
-                };
+                const tag = $('.p-header-category-breadcrumbs li a').last().text();
+                item.category = tag;
+                item.title = `[${tag}] ${item.title}`;
+                return item;
             })
-            .get()
+        )
     );
 
     ctx.state.data = {
-        title: '読売新聞' + formattedCategoryName,
+        title: $('head title').text(),
         link: url,
+        image: 'https://www.yomiuri.co.jp/apple-touch-icon.png',
         item: items,
     };
 };

package/lib/v2/yomiuri/radar.js

@@ -3,10 +3,10 @@ module.exports = {
         _name: '読売新聞',
         www: [
             {
-                title: '読売新聞',
+                title: '新聞',
                 docs: 'https://docs.rsshub.app/traditional-media.html#du-mai-xin-wen',
-                source: ['/:category'],
-                target: '/yomiuri/:category',
+                source: ['/:category?'],
+                target: '/yomiuri/:category?',
             },
         ],
     },

package/lib/v2/youtube/channel.js

@@ -4,14 +4,13 @@ const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     if (!config.youtube || !config.youtube.key) {
-        throw 'YouTube RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>';
+        throw Error('YouTube RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
     }
     const id = ctx.params.id;
     const embed = !ctx.params.embed;
 
-    // taken from https://webapps.stackexchange.com/a/101153
-    if (!/^UC[\w-]{21}[AQgw]$/.test(id)) {
-        throw `Invalid YouTube channel ID. \nYou may want to use <a href="/youtube/user/${id}">/youtube/user/${id}</a> instead.`;
+    if (!utils.isYouTubeChannelId(id)) {
+        throw Error(`Invalid YouTube channel ID. \nYou may want to use <code>/youtube/user/:id</code> instead.`);
     }
 
     const playlistId = (await utils.getChannelWithId(id, 'contentDetails', ctx.cache)).data.items[0].contentDetails.relatedPlaylists.uploads;

package/lib/v2/youtube/charts.js

@@ -0,0 +1,97 @@
+const got = require('@/utils/got');
+const { renderDescription } = require('./utils');
+const config = require('@/config').value;
+
+module.exports = async (ctx) => {
+    const contentMap = {
+        TopArtists: {
+            contentKey: 'artists',
+            viewKey: 'artistViews',
+            title: 'Top artists',
+        },
+        TopSongs: {
+            contentKey: 'trackTypes',
+            viewKey: 'trackViews',
+            title: 'Top songs',
+        },
+        TopVideos: {
+            contentKey: 'videos',
+            viewKey: 'videoViews',
+            title: 'Top music videos',
+        },
+        TrendingVideos: {
+            contentKey: 'videos',
+            viewKey: 'videoViews',
+            title: 'Trending',
+        },
+    };
+
+    const { category = 'TopVideos', country } = ctx.params;
+    const embed = !ctx.params.embed;
+
+    const { content } = await ctx.cache.tryGet(
+        `youtube:charts:${country ? country : 'global'}`,
+        async () => {
+            const { data } = await got.post('https://charts.youtube.com/youtubei/v1/browse', {
+                searchParams: {
+                    alt: 'json',
+                    key: 'AIzaSyCzEW7JUJdSql0-2V4tHUb6laYm4iAE_dM',
+                },
+                json: {
+                    browseId: 'FEmusic_analytics_charts_home',
+                    context: {
+                        capabilities: {},
+                        client: {
+                            clientName: 'WEB_MUSIC_ANALYTICS',
+                            clientVersion: '0.2',
+                            experimentIds: [],
+                            experimentsToken: '',
+                            gl: 'US',
+                            hl: 'en',
+                            theme: 'MUSIC',
+                        },
+                        request: {
+                            internalExperimentFlags: [],
+                        },
+                    },
+                    query: `chart_params_type=WEEK&perspective=CHART&flags=viral_video_chart&selected_chart=TRACKS&chart_params_id=weekly:0:0${country ? `:${country}` : ''}`,
+                },
+            });
+            return data.contents.sectionListRenderer.contents[0].musicAnalyticsSectionRenderer;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const { entityId } = content.perspectiveMetadata;
+    let items;
+
+    if (category === 'TopArtists') {
+        items = content[contentMap[category].contentKey][0][contentMap[category].viewKey].map((item) => ({
+            title: item.name,
+            link: `https://charts.youtube.com/artist/${encodeURIComponent(item.id)}`,
+            guid: `youtube:charts:${category}:${entityId}:${item.id}`,
+        }));
+    } else {
+        items = content[contentMap[category].contentKey][category !== 'TrendingVideos' ? 0 : 1][contentMap[category].viewKey].map((item) => {
+            const videoId = category === 'TopSongs' ? item.encryptedVideoId : item.id;
+            const author = item.artists
+                .filter((a) => a.name)
+                .map((artist) => artist.name)
+                .join(', ');
+            return {
+                title: `${item.title ?? item.name} - ${author}`,
+                description: renderDescription(embed, videoId, item.thumbnail?.thumbnails.pop().url, ''),
+                link: `https://www.youtube.com/watch?v=${videoId}`,
+                guid: `youtube:charts:${category}:${entityId}:${item.id}`,
+                author,
+            };
+        });
+    }
+
+    ctx.state.data = {
+        title: `YouTube Music Charts - ${contentMap[category].title}`,
+        link: `https://charts.youtube.com/charts/${category}/${country ? country : 'global'}`,
+        item: items,
+    };
+};