rsshub 1.0.0-master.fe99193 → 1.0.0-master.fe9c82a

package/lib/v2/xiaohongshu/util.js

@@ -1,6 +1,7 @@
 const config = require('@/config').value;
+const logger = require('@/utils/logger');
 
-const getContent = (url, cache) =>
+const getUser = (url, cache) =>
     cache.tryGet(
         url,
         async () => {
@@ -8,15 +9,50 @@ const getContent = (url, cache) =>
             try {
                 const page = await browser.newPage();
                 await page.setRequestInterception(true);
+                let otherInfo = '';
+                let user_posted = '';
+                let collect = '';
                 page.on('request', (request) => {
-                    request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+                    request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' || request.resourceType() === 'other' ? request.continue() : request.abort();
                 });
-                page.on('response', (response) => {
-                    if (response.url().includes('captcha.fengkongcloud.cn')) {
-                        page.close();
-                        throw Error('滑块验证');
+                page.on('response', async (response) => {
+                    const request = response.request();
+                    if (request.url().includes('/api/sns/web/v1/user/otherinfo')) {
+                        otherInfo = await response.json();
+                    }
+                    if (request.url().includes('/api/sns/web/v1/user_posted')) {
+                        user_posted = await response.json();
+                    }
+                    if (request.url().includes('/api/sns/web/v2/note/collect/page')) {
+                        collect = await response.json();
                     }
                 });
+                logger.debug(`Requesting ${url}`);
+                await page.goto(url);
+                await page.waitForSelector('.feeds-container:nth-child(1) .note-item');
+                await page.click('div.reds-tab-item:nth-child(2)');
+                await page.waitForSelector('.feeds-container:nth-child(2) .note-item');
+                return { otherInfo, user_posted, collect };
+            } finally {
+                browser.close();
+            }
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+const getBoard = (url, cache) =>
+    cache.tryGet(
+        url,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            try {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+                });
+                logger.debug(`Requesting ${url}`);
                 await page.goto(url);
                 await page.waitForSelector('.pc-container');
                 const initialSsrState = await page.evaluate(() => window.__INITIAL_SSR_STATE__);
@@ -30,5 +66,6 @@ const getContent = (url, cache) =>
     );
 
 module.exports = {
-    getContent,
+    getUser,
+    getBoard,
 };

package/lib/v2/xiaomiyoupin/crowdfunding.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const base_url = 'https://m.xiaomiyoupin.com';
+module.exports = async (ctx) => {
+    const resp = await got('https://home.mi.com/lasagne/page/5');
+    const site_url = resp.data.redirect.location;
+
+    const urlParams = new URLSearchParams(site_url);
+    const pageid = urlParams.get('pageid');
+    const sign = urlParams.get('sign');
+
+    // 1. fetchPageData
+    const pageData = await got(`${base_url}/mtop/navi/venue/page?page_id=${pageid}&pdl=jianyu&sign=${sign}`);
+    const crowd_funding_floor = pageData.data.data.floors.find((floor) => floor.module_key === 'crowding');
+    const query_list = crowd_funding_floor.query_list;
+
+    // 2. fetchFloorDetailData
+    const floor_detail_data = await got.post(`${base_url}/mtop/navi/venue/batch?page_id=${pageid}&pdl=jianyu&sign=${sign}`, {
+        json: {
+            query_list,
+        },
+    });
+
+    const goodsList = floor_detail_data.data.data.result_list[0].list;
+    const items = goodsList.map((e) => {
+        const goods = e.value.goods;
+        return {
+            title: goods.name,
+            guid: `xiaomiyoupin:${goods.gid}`,
+            description: art(path.join(__dirname, 'templates/goods.art'), goods),
+            link: goods.jump_url,
+            pubDate: new Date(goods.fist_release_time * 1000).toUTCString(),
+        };
+    });
+    ctx.state.data = {
+        title: '小米有品众筹',
+        link: site_url,
+        description: '小米有品众筹',
+        item: items,
+    };
+};

package/lib/v2/xiaomiyoupin/latest.js

@@ -0,0 +1,15 @@
+const got = require('@/utils/got');
+const { parseModule, parseFloorItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const response = await got('https://m.xiaomiyoupin.com/homepage/main/v1005');
+    const floors = parseModule(response.data.data.homepage.floors, 'product_hot');
+    const items = parseFloorItem(floors);
+
+    ctx.state.data = {
+        title: '小米有品每日上新',
+        link: 'https://m.xiaomiyoupin.com/w/newproduct?pageid=1605',
+        description: '小米有品每日上新',
+        item: items,
+    };
+};

package/lib/v2/xiaomiyoupin/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/crowdfunding': ['bigfei'],
+    '/latest': ['xyqfer', 'DIYgod', 'bigfei'],
+};

package/lib/v2/xiaomiyoupin/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'xiaomiyoupin.com': {
+        _name: '小米有品',
+        '.': [
+            {
+                title: '小米有品众筹',
+                docs: 'https://docs.rsshub.app/shopping.html#xiao-mi-you-pin-xiao-mi-you-pin-zhong-chou',
+                source: ['/'],
+                target: '/xiaomiyoupin/crowdfunding',
+            },
+            {
+                title: '小米有品每日上新',
+                docs: 'https://docs.rsshub.app/shopping.html#xiao-mi-you-pin-xiao-mi-you-pin-mei-ri-shang-xin',
+                source: ['/'],
+                target: '/xiaomiyoupin/latest',
+            },
+        ],
+    },
+};

package/lib/v2/xiaomiyoupin/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/crowdfunding', require('./crowdfunding'));
+    router.get('/latest', require('./latest'));
+};

package/lib/v2/xiaomiyoupin/templates/goods.art

@@ -0,0 +1,27 @@
+<figure>
+    <img src="{{ pic_url || img_square || imgs.img800 }}" alt="{{ alt }}" loading="lazy" style="display:block; margin-left:auto; margin-right:auto;" width:100%;>
+    {{ if videos_url }}
+    <video controls
+        playsinline="true"
+        webkit-playsinline="true"
+        x5-playsinline="true"
+        x5-video-player-type="h5"
+        x5-video-orientation="landscape|portrait"
+        x5-video-player-fullscreen="true"
+        x-webkit-airplay="allow"
+        preload="metadata"
+        poster="{{ pic_url || imgs.img800 }}">
+        <source src="{{ videos_url[0] }}" type="video/mp4" />
+    </video>
+    {{/if}}
+
+    {{ if name }}
+    <figcaption>
+        <div class="caption">{{@ name }}</div>
+        <div class="credit">{{@ summary }}</div>
+        <div class="price">原始价格：{{@ market_price / 100 }}元</div>
+        <div class="price">实际价格：{{@ (price_min || flash_price) / 100 }}元</div>
+    </figcaption>
+    {{/if}}
+
+</figure>

package/lib/v2/xiaomiyoupin/utils.js

@@ -0,0 +1,21 @@
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const parseModule = (floors, module_key) => floors.filter((floor) => floor.module_key === module_key)[0];
+
+const parseFloorItem = (floor) =>
+    floor.data.items.map((item) => {
+        const i = item.item;
+        return {
+            title: i.name,
+            link: i.jump_url,
+            guid: `xiaomiyoupin:${i.gid}`,
+            description: art(path.join(__dirname, 'templates/goods.art'), i),
+            pubDate: (i.start || i.start_time) * 1000,
+        };
+    });
+
+module.exports = {
+    parseModule,
+    parseFloorItem,
+};

package/lib/v2/xunhupay/index.js

@@ -0,0 +1,24 @@
+const buildData = require('@/utils/common-config');
+const baseUrl = 'https://www.xunhupay.com';
+
+module.exports = async (ctx) => {
+    const link = `${baseUrl}/blog.html`;
+    ctx.state.data = await buildData({
+        link,
+        url: link,
+        title: `%title%`,
+        description: `%description%`,
+        params: {
+            title: '博客',
+            description: '虎皮椒-博客',
+        },
+        item: {
+            item: '.blog-post > article',
+            title: `$('h5').text()`,
+            link: `$('a').attr('href')`,
+            description: `$('.content').text()`,
+            pubDate: `parseDate($('.date').text(), 'YYYY-MM-DD')`,
+            guid: Buffer.from(`$('a').attr('href')`).toString('base64'),
+        },
+    });
+};

package/lib/v2/xunhupay/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/blog': ['Joey'],
+};

package/lib/v2/xunhupay/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'xunhupay.com': {
+        _name: '虎皮椒',
+        www: [
+            {
+                title: '博客',
+                docs: 'https://docs.rsshub.app/blog.html#hu-pi-jiao',
+                source: ['/blog'],
+                target: '/xunhupay/blog',
+            },
+        ],
+    },
+};

package/lib/v2/xunhupay/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/blog', require('./index'));
+};

package/lib/v2/xys/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/new': ['wenzhenl'],
+};

package/lib/v2/xys/new.js

@@ -0,0 +1,74 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const iconv = require('iconv-lite');
+const path = require('path');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'http://www.xys.org';
+    const currentUrl = `${rootUrl}/new.html`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+        responseType: 'buffer',
+    });
+
+    // 转码
+    const data = iconv.decode(response.data, 'gb2312');
+
+    const $ = cheerio.load(data);
+
+    let items = $('li a')
+        .slice(4, ctx.query.limit ? parseInt(ctx.query.limit) : 30)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            let link = item.attr('href');
+            /^https?:\/\//.test(link) || (link = rootUrl + '/' + link.replace(/^\//, ''));
+            let date = item.parent().text().trim().slice(0, 8);
+            date = parseDate(date, 'YY.MM.DD');
+            return {
+                title: item.text(),
+                link,
+                pubDate: date,
+            };
+        });
+
+    items = await Promise.all(
+        items
+            .filter((item) => !item.link.endsWith('.zip'))
+            .map((item) =>
+                ctx.cache.tryGet(item.link, async () => {
+                    const youTube = /(?:https?:\/\/)?(?:www\.)?youtu\.?be(?:\.com)?\/?.*(?:watch|embed)?(?:.*v=|v\/|\/)([\w\-_]+)&?/g;
+                    const matchYoutube = item.link.match(youTube);
+
+                    if (matchYoutube) {
+                        item.description = art(path.join(__dirname, 'templates/desc.art'), { youTube: item.link.slice(32) });
+                    } else {
+                        const detailResponse = await got({
+                            method: 'get',
+                            url: item.link,
+                            responseType: 'buffer',
+                        });
+
+                        // 转码
+                        const detailData = iconv.decode(detailResponse.data, 'gb2312');
+
+                        const content = cheerio.load(detailData);
+
+                        item.description = content.text().replace(/\n/g, '<br>\n');
+                    }
+
+                    return item;
+                })
+            )
+    );
+
+    ctx.state.data = {
+        title: '新语丝 - 新到资料',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/xys/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'xys.org': {
+        _name: '新语丝',
+        '.': [
+            {
+                title: '新到资料',
+                docs: 'https://docs.rsshub.app/blog.html#xin-yu-si',
+                source: ['/', '/new.html'],
+                target: '/xys/new',
+            },
+        ],
+    },
+};

package/lib/v2/xys/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/new', require('./new'));
+};

package/lib/v2/xys/templates/desc.art

@@ -0,0 +1,3 @@
+{{ if youTube }}
+<iframe width="560" height="315" src="https://www.youtube-nocookie.com/embed/{{ youTube }}" frameborder="0" allowfullscreen></iframe>
+{{ /if }}

package/lib/v2/yahoo/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:region/:category?': ['KeiLongW'],
+};

package/lib/{routes/yahoo-news → v2/yahoo/news}/index.js

@@ -1,16 +1,20 @@
 const got = require('@/utils/got');
 const parser = require('@/utils/rss-parser');
 const cheerio = require('cheerio');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
-    const region = ctx.params.region === 'en' ? '' : ctx.params.region.toLowerCase() + '.';
+    const region = ctx.params.region === 'en' ? '' : ctx.params.region.toLowerCase();
+    if (!isValidHost(region) && region !== '') {
+        throw Error('Invalid region');
+    }
     const category = ctx.params.category ? ctx.params.category.toLowerCase() : '';
-    const rssUrl = `https://${region}news.yahoo.com/rss/${category}`;
+    const rssUrl = `https://${region ? region + '.' : ''}news.yahoo.com/rss/${category}`;
     const feed = await parser.parseURL(rssUrl);
     const title = feed.title;
     const link = feed.link;
     const description = feed.description;
-    const filteredItems = feed.items.filter((item) => !item.link.includes('promotions') && item.link.includes('yahoo.com'));
+    const filteredItems = feed.items.filter((item) => !item.link.includes('promotions') && new URL(item.link).hostname.match(/.*\.yahoo\.com$/));
     const items = await Promise.all(
         filteredItems.map(async (item) => {
             const description = await ctx.cache.tryGet(item.link, async () => {
@@ -20,10 +24,8 @@ module.exports = async (ctx) => {
                 });
 
                 const $ = cheerio.load(response.data);
-                const $descriptionRoot = cheerio.load('<div><div class="img-root"></div><div class="content-root"></div></div>');
-                $descriptionRoot('div.img-root').append($('article img:not(header img)'));
-                $descriptionRoot('div.content-root').append($('article p').removeAttr('content'));
-                return $descriptionRoot.html();
+                $('.caas-content-byline-wrapper, .caas-xray-wrapper, .caas-header, .caas-readmore').remove();
+                return $('.caas-content-wrapper').html();
             });
             const single = {
                 title: item.title,
@@ -31,7 +33,7 @@ module.exports = async (ctx) => {
                 pubDate: item.pubDate,
                 link: item.link,
             };
-            return Promise.resolve(single);
+            return single;
         })
     );
 

package/lib/v2/yahoo/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'yahoo.com': {
+        _name: 'Yahoo',
+        '.': [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yahoo',
+                source: ['/'],
+                target: '/yahoo/news/:region/:category?',
+            },
+        ],
+    },
+};

package/lib/v2/yahoo/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/news/:region/:category?', require('./news/index'));
+};

package/lib/v2/ycwb/index.js

@@ -0,0 +1,82 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const node = ctx.params.node ?? 1;
+    const currentUrl = `https://6api.ycwb.com/app_if/jy/getArticles?nodeid=${node}&pagesize=15`;
+
+    const { data: response } = await got(currentUrl);
+
+    const list = response.artiles.map((item) => ({
+        title: item.TITLE,
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            thumb: item.PICLINKS,
+            description: item.ABSTRACT,
+        }),
+        pubDate: timezone(parseDate(item.PUBTIME), +8),
+        link: item.PUBURL,
+        nodeName: item.NODENAME,
+    }));
+
+    let nodeName = '';
+    let nodeLink = '';
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                const $comments = content('.main_article')
+                    .contents()
+                    .filter(function () {
+                        return this.nodeType === 8;
+                    });
+                $comments.each(function () {
+                    // Remove useless comments
+                    if (this.data.match(/audioPlayer|audio-box/)) {
+                        this.data = '';
+                    }
+                    // Filter author from comments
+                    if (this.data.match(/author/)) {
+                        item.author = this.data.split('<author>')[1].split('</author>')[0];
+                    }
+                });
+
+                nodeName = nodeName === '' ? item.nodeName : nodeName;
+
+                // Filter node link from content('.path a')
+                const children = content('.path').children('a');
+                for (const child of children) {
+                    if (content(child).text() === nodeName && nodeLink === '') {
+                        nodeLink = content(child).attr('href');
+                    }
+                }
+
+                content('span').removeAttr('style').removeAttr('class');
+                content('img').removeAttr('style').removeAttr('class').removeAttr('placement').removeAttr('data-toggle').removeAttr('trigger').removeAttr('referrerpolicy');
+                content('br').removeAttr('style').removeAttr('class');
+                content('p').removeAttr('style').removeAttr('class');
+                content('.space10, .ddf').remove();
+
+                item.description += content('.main_article').html() ?? '';
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `羊城晚报金羊网 - ${nodeName}`,
+        link: String(nodeLink === '' ? 'https://www.ycwb.com/' : nodeLink),
+        item: items,
+    };
+};

package/lib/v2/ycwb/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:node': ['TimWu007'],
+};

package/lib/v2/ycwb/radar.js

@@ -0,0 +1,12 @@
+module.exports = {
+    'xwlb.com.cn': {
+        _name: '羊城晚报金羊网',
+        '.': [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yang-cheng-wan-bao-jin-yang-wang',
+                source: ['/'],
+            },
+        ],
+    },
+};

package/lib/v2/ycwb/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:node', require('./index'));
+};

package/lib/v2/ycwb/templates/description.art

@@ -0,0 +1,7 @@
+{{ if thumb }}
+<img src="{{ thumb }}"><br>
+{{ /if }}
+{{ if description }}
+    <blockquote><p>{{ description }}</p></blockquote>
+    <br>
+{{ /if }}

package/lib/v2/yicai/utils.js

@@ -28,7 +28,7 @@ module.exports = {
             }),
         }));
 
-        return await Promise.all(
+        return Promise.all(
             items.map((item) =>
                 tryGet(item.link, async () => {
                     const detailResponse = await got({

package/lib/v2/ymgal/article.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+const host = 'https://www.ymgal.games';
+
+const types = {
+    news: '?type=NEWS&page=1',
+    column: '?type=COLUMN&page=1',
+};
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type || 'all';
+
+    const link = `${host}/co/topic/list` + types[type];
+    let data = [];
+    if (type === 'all') {
+        await Promise.all(
+            Object.values(types).map(async (type) => {
+                const response = await got(`${host}/co/topic/list${type}`);
+                data.push.apply(data, response.data.data);
+            })
+        );
+        data = data.sort((a, b) => b.publishTime - a.publishTime).slice(0, 10);
+    } else {
+        const response = await got(link);
+        data = response.data.data;
+    }
+
+    const items = await Promise.all(
+        data.map((item) => {
+            const itemUrl = host + '/co/article/' + item.topicId;
+            return ctx.cache.tryGet(itemUrl, async () => {
+                const result = await got(itemUrl);
+                const $ = cheerio.load(result.data);
+                const description = $('article').html().trim();
+                return {
+                    title: item.title,
+                    link: itemUrl,
+                    pubDate: timezone(parseDate(item.publishTime), 8),
+                    description,
+                };
+            });
+        })
+    );
+
+    let info = '全部文章';
+    if (type === 'news') {
+        info = '资讯';
+    } else if (type === 'column') {
+        info = '专栏';
+    }
+
+    ctx.state.data = {
+        title: `月幕 Galgame - ${info}`,
+        link: `${host}/co/article`,
+        description: `月幕 Galgame - ${info}`,
+        item: items,
+    };
+};