rsshub 1.0.0-master.fe69819 → 1.0.0-master.febd623

package/lib/v2/wp-china/news.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const config = {
+    latest: 'list-35b3413e4c03411d8f3726162570804f',
+    stories: 'list-7224ef82da1b498eb24046a6c56ff566',
+};
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'latest';
+
+    const rootUrl = 'http://www.wp-china.com';
+    const currentUrl = `${rootUrl}/f/${config[category]}.html`;
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('.new_list')
+        .slice(0, 10)
+        .map((_, item) => {
+            item = $(item);
+            return {
+                title: item.children('.fb').text(),
+                link: `${rootUrl}${item.parent().attr('href')}`,
+                pubDate: timezone(parseDate(item.children('.grey').text()), +8),
+            };
+        })
+        .get();
+
+    const items = await Promise.all(
+        list.map(
+            async (item) =>
+                await ctx.cache.tryGet(item.link, async () => {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+                    const content = cheerio.load(detailResponse.data);
+
+                    item.description = content('.wb100').html();
+
+                    return item;
+                })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text().replace(' - Powered By gobestsoft', ''),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/wp-china/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'wp-china.com': {
+        _name: '中国工人出版社',
+        '.': [
+            {
+                title: '新闻中心',
+                docs: 'https://docs.rsshub.app/new-media.html#zhong-guo-gong-ren-chu-ban-she',
+                source: ['/'],
+                target: '/wp-china/news',
+            },
+        ],
+    },
+};

package/lib/v2/wp-china/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/news/:category?', require('./news'));
+};

package/lib/v2/wsj/index.js

@@ -0,0 +1,141 @@
+const parser = require('@/utils/rss-parser');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const categoryToXMLFileName = {
+    opinion: 'RSSOpinion.xml',
+    world_news: 'RSSWorldNews.xml',
+    us_bussiness: 'WSJcomUSBusiness.xml',
+    market_news: 'RSSMarketsMain.xml',
+    technology: 'RSSWSJD.xml',
+    lifestyle: 'RSSLifestyle.xml',
+};
+
+const categoryToName = {
+    opinion: 'Opinion',
+    world_news: 'World News',
+    us_bussiness: 'U.S. Business',
+    market_news: 'Markets News',
+    technology: "Technology: What's News",
+    lifestyle: 'Lifestyle',
+};
+
+module.exports = async (ctx) => {
+    const language = ctx.params.lang;
+    const category = ctx.params.category;
+    let rssUrl;
+    switch (language) {
+        case 'en-us':
+            rssUrl = `https://feeds.a.dj.com/rss/${categoryToXMLFileName[category]}`;
+            break;
+        case 'zh-cn':
+            // Doesn't support categorical subscribtion
+            rssUrl = `https://cn.wsj.com/zh-hans/rss/`;
+            break;
+        case 'zh-tw':
+            rssUrl = `https://cn.wsj.com/zh-hant/rss/`;
+            break;
+        default:
+            // Doesn't support other languages (e.g. ja) for now
+            throw Error(`Language ${language} is not supported`);
+    }
+
+    const feed = await parser.parseURL(rssUrl);
+    const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
+
+    const items = await Promise.all(
+        feed.items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                // Fetch the AMP version
+                const url = item.link.replace(/(?<=^https:\/\/\w+\.wsj\.com)/, '/amp');
+                const response = await got({
+                    url,
+                    method: 'get',
+                    headers: {
+                        'User-Agent': chromeMobileUserAgent,
+                    },
+                });
+                const html = response.body;
+                const $ = cheerio.load(html);
+                const content = $('.articleBody > section');
+
+                // Cover
+                const cover = $('.articleLead > div.is-lead-inset > div.header > .img-header > div.image-container > amp-img > img');
+
+                if (cover.length > 0) {
+                    $(`<img src=${cover[0].attribs.content}>`).insertBefore(content[0].childNodes[0]);
+                    $(cover).remove();
+                }
+
+                // Summary
+                const summary = $('head > meta[name="description"]').attr('content');
+
+                // Metadata (categories & updatedAt)
+                const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
+                const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
+                const author = $('.author > a[rel="author"]').html();
+
+                const categories = $('meta[name="keywords"]')
+                    .attr('content')
+                    .split(',')
+                    .map((c) => c.trim());
+
+                // Images
+                content.find('amp-img').each((i, e) => {
+                    const img = $(`<img width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}" alt="${e.attribs.alt}">`);
+
+                    // Caption follows, no need to handle caption
+                    $(img).insertBefore(e);
+                    $(e).remove();
+                });
+
+                // iframes (youtube videos and interactive elements)
+                content.find('amp-iframe').each((i, e) => {
+                    const iframe = $(`<iframe width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}">`);
+                    $(iframe).insertBefore(e);
+                    $(e).remove();
+                });
+
+                // Remove unwanted DOMs
+                const unwanted_element_selectors = ['amp-ad', '.wsj-ad'];
+                unwanted_element_selectors.forEach((selector) => {
+                    content.find(selector).each((i, e) => {
+                        $(e).remove();
+                    });
+                });
+
+                // Paywall
+                content.find('.paywall').each((i, e) => {
+                    // Caption follows, no need to handle caption
+                    $(e.childNodes).insertBefore(e);
+                    $(e).remove();
+                });
+
+                return {
+                    title: item.title,
+                    id: item.guid,
+                    pubDate: parseDate(publishedAt),
+                    updated: parseDate(updatedAt),
+                    author,
+                    link: item.link,
+                    summary,
+                    description: content.html(),
+                    category: categories,
+                    icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
+                    logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: categoryToName[category],
+        link: feed.link,
+        description: feed.description,
+        item: items,
+        language: feed.language,
+        icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
+        logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
+    };
+};

package/lib/v2/wsj/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:lang/:category?': ['oppilate'],
+};

package/lib/v2/wsj/radar.js

@@ -0,0 +1,21 @@
+module.exports = {
+    'wsj.com': {
+        _name: '华尔街日报 The Wall Street Journal (WSJ)',
+        cn: [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#hua-er-jie-ri-bao-the-wall-street-journal-wsj',
+                souce: '/',
+                target: '/wsj/zh-cn',
+            },
+        ],
+        www: [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#hua-er-jie-ri-bao-the-wall-street-journal-wsj',
+                souce: '/',
+                target: '/wsj/en-us',
+            },
+        ],
+    },
+};

package/lib/v2/wsj/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:lang/:category?', require('./index'));
+};

package/lib/v2/xiaohongshu/board.js

@@ -0,0 +1,28 @@
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { getContent } = require('./util');
+
+module.exports = async (ctx) => {
+    const url = `https://www.xiaohongshu.com/board/${ctx.params.board_id}`;
+    const main = await getContent(url, ctx.cache);
+
+    const albumInfo = main.albumInfo;
+    const title = albumInfo.name;
+    const description = albumInfo.desc;
+
+    const list = main.notesDetail;
+    const resultItem = list.map((item) => ({
+        title: item.title,
+        link: `https://www.xiaohongshu.com/discovery/item/${item.id}`,
+        description: `<img src ="${item.cover.url.split('?imageView2')[0]}"><br>${item.title}`,
+        author: item.user.nickname,
+        pubDate: timezone(parseDate(item.time), 8),
+    }));
+
+    ctx.state.data = {
+        title,
+        link: url,
+        item: resultItem,
+        description,
+    };
+};

package/lib/v2/xiaohongshu/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/board/:board_id': ['lotosbin'],
+    '/user/:user_id/:category': ['lotosbin'],
+};

package/lib/v2/xiaohongshu/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'xiaohongshu.com': {
+        _name: '小红书',
+        '.': [
+            {
+                title: '用户笔记',
+                docs: 'https://docs.rsshub.app/social-media.html#xiao-hong-shu',
+                source: '/user/profile/:user_id',
+                target: '/xiaohongshu/user/:user_id/notes',
+            },
+            {
+                title: '用户专辑',
+                docs: 'https://docs.rsshub.app/social-media.html#xiao-hong-shu',
+                source: '/user/profile/:user_id',
+                target: '/xiaohongshu/user/:user_id/album',
+            },
+            {
+                title: '专辑',
+                docs: 'https://docs.rsshub.app/social-media.html#xiao-hong-shu',
+                source: '/board/:board_id',
+                target: '/xiaohongshu/board/:board_id',
+            },
+        ],
+    },
+};

package/lib/v2/xiaohongshu/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/board/:board_id', require('./board'));
+    router.get('/user/:user_id/:category', require('./user'));
+};

package/lib/v2/xiaohongshu/user.js

@@ -0,0 +1,46 @@
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { getContent } = require('./util');
+
+module.exports = async (ctx) => {
+    const userId = ctx.params.user_id;
+    const category = ctx.params.category;
+    const url = `https://www.xiaohongshu.com/user/profile/${userId}`;
+
+    const main = await getContent(url, ctx.cache);
+    const userDetail = main.userDetail;
+    const description = `${userDetail.nickname} • 小红书 / RED`;
+    if (category === 'notes') {
+        const list = main.notesDetail;
+        const resultItem = list.map((item) => ({
+            title: item.title,
+            link: `https://www.xiaohongshu.com/discovery/item/${item.id}`,
+            description: `<img src ="${item.cover.url.split('?imageView2')[0]}"><br>${item.title}`,
+            author: item.user.nickname,
+            pubDate: timezone(parseDate(item.time), 8),
+        }));
+
+        const title = `小红书-${userDetail.nickname}-笔记`;
+        ctx.state.data = {
+            title,
+            link: url,
+            item: resultItem,
+            description,
+        };
+    } else if (category === 'album') {
+        const list = main.albumDetail;
+        const resultItem = list.map((item) => ({
+            title: item.title,
+            link: `https://www.xiaohongshu.com/board/${item.id}`,
+            description: item.images.map((it) => `<img src ="${it.split('?imageView2')[0]}">`).join(`<br>`),
+        }));
+
+        const title = `小红书-${userDetail.nickname}-专辑`;
+        ctx.state.data = {
+            title,
+            link: url,
+            item: resultItem,
+            description,
+        };
+    }
+};

package/lib/v2/xiaohongshu/util.js

@@ -0,0 +1,28 @@
+const config = require('@/config').value;
+
+const getContent = (url, cache) =>
+    cache.tryGet(
+        url,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            try {
+                const page = await browser.newPage();
+                await page.goto(url);
+                await page.waitForSelector('.pc-container');
+                let content = await page.content();
+
+                content = content.replace('"RedAppLayout":undefined,', ''); // fix json parse
+                const regex = /_SSR_STATE__\S*=([\s\S]+)<\/script>/gm;
+                const match = regex.exec(content);
+                return JSON.parse(match[1]).Main;
+            } finally {
+                browser.close();
+            }
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+module.exports = {
+    getContent,
+};

package/lib/v2/xiaoyuzhou/maintainer.js

@@ -1,4 +1,4 @@
 module.exports = {
     '/': ['prnake', 'Maecenas'],
-    '/podcast/:id': ['hondajojo'],
+    '/podcast/:id': ['hondajojo', 'jtsang4'],
 };

package/lib/v2/xiaoyuzhou/pickup.js

@@ -58,7 +58,7 @@ const ProcessFeed = async (ctx) => {
         const itunes_item_image = item.episode.image ? item.episode.image.picUrl : item.episode.podcast.image ? item.episode.podcast.image.picUrl : '';
         const link = `https://www.xiaoyuzhoufm.com/episode/${eid}`;
         const pubDate = item.pubDate;
-        const enclosure_length = item.episode.duration;
+        const itunes_duration = item.episode.duration;
         const enclosure_url = item.episode.enclosure.url;
         const desc = `<p><strong>${item.comment.author.nickname}：</strong>${item.comment.text}</p><hr>` + item.episode.shownotes;
         const author = item.episode.podcast.author;
@@ -70,7 +70,7 @@ const ProcessFeed = async (ctx) => {
             author,
             pubDate,
             enclosure_url,
-            enclosure_length,
+            itunes_duration,
             itunes_item_image,
             enclosure_type: 'audio/mpeg',
         };

package/lib/v2/xiaoyuzhou/podcast.js

@@ -16,12 +16,12 @@ module.exports = async (ctx) => {
     const episodes = page_data.props.pageProps.podcast.episodes.map((item) => ({
         title: item.title,
         enclosure_url: item.enclosure.url,
-        enclosure_length: item.duration,
+        itunes_duration: item.duration,
         enclosure_type: 'audio/mpeg',
         link: `https://www.xiaoyuzhoufm.com/episode/${item.eid}`,
         pubDate: parseDate(item.pubDate),
         description: item.shownotes,
-        itunes_item_image: item.image.smallPicUrl,
+        itunes_item_image: (item.image || item.podcast?.image)?.smallPicUrl,
     }));
 
     ctx.state.data = {

package/lib/v2/xiaozhuanlan/column.js

@@ -0,0 +1,52 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const md = require('markdown-it')({
+    linkify: true,
+});
+const baseUrl = 'https://xiaozhuanlan.com';
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const response = await got(`${baseUrl}/${id}`);
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.xzl-topic-item')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            item.find('.topic-has-suggested-item').remove();
+            return {
+                title: item.find('h3').text().trim(),
+                link: new URL(item.find('.topic-body-link').attr('href'), baseUrl).href,
+                author: item.find('.topic-header .xzl-author-lockup').text().trim(),
+                pubDate: parseDate(item.find('.topic-header .timeago').attr('title')),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const $ = cheerio.load(detailResponse.data);
+
+                item.description = md.render($('.hidden_markdown_body').attr('data-summary'));
+                if ($('.topic-tags')) {
+                    item.category = $('.topic-tags label')
+                        .toArray()
+                        .map((l) => $(l).text());
+                }
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text().trim(),
+        link: `${baseUrl}/${id}`,
+        description: $('meta[name=description]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/xiaozhuanlan/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/column/:id': ['TonyRL'],
+};

package/lib/v2/xiaozhuanlan/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'xiaozhuanlan.com': {
+        _name: '小专栏',
+        '.': [
+            {
+                title: '专栏',
+                docs: 'https://docs.rsshub.app/new-media.html#xiao-zhuan-lan',
+                source: '/:id',
+                target: '/xiaozhuanlan/column/:id',
+            },
+        ],
+    },
+};

package/lib/v2/xiaozhuanlan/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/column/:id', require('./column'));
+};

package/lib/v2/xidian/jwc.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const baseUrl = 'https://jwc.xidian.edu.cn';
+
+module.exports = async (ctx) => {
+    const { category = 'tzgg' } = ctx.params;
+    const url = `${baseUrl}/${category}.htm`;
+    const response = await got(url, {
+        headers: {
+            referer: baseUrl,
+        },
+        https: {
+            rejectUnauthorized: false,
+        },
+    });
+    const $ = cheerio.load(response.data);
+
+    let items = $('.list ul li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: new URL(item.find('a').attr('href'), baseUrl).href,
+                pubDate: parseDate(item.find('.con span').text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link, {
+                    headers: {
+                        referer: url,
+                    },
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
+                const content = cheerio.load(detailResponse.data);
+                content('.tit, .zd, #div_vote_id').remove();
+                item.description = content('.con').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/xidian/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/jwc/:category?': ['ShadowySpirits'],
+};

package/lib/v2/xidian/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'xidian.edu.cn': {
+        _name: '西安电子科技大学',
+        jwc: [
+            {
+                title: '教务处',
+                docs: 'https://docs.rsshub.app/university.html#xi-an-dian-zi-ke-ji-da-xue',
+                source: ['/:category'],
+                target: (params) => `/xidian/jwc${params.category ? `/${params.category}` : ''}`,
+            },
+        ],
+    },
+};

package/lib/v2/xidian/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/jwc/:category?', require('./jwc'));
+};

package/lib/v2/xmanhua/index.js

@@ -0,0 +1,74 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const uid = ctx.params.uid;
+    const host = 'https://xmanhua.com';
+    const url = `https://xmanhua.com/${uid}/`;
+    const response = await got({
+        method: 'get',
+        url,
+        headers: {
+            Referer: host,
+        },
+    });
+
+    const data = response.data;
+    const $ = cheerio.load(data);
+    const list = $('div #chapterlistload').find('.detail-list-form-item');
+    // 作者
+    const autherName = $('body > div.detail-info-1 > div > div > p.detail-info-tip > span:nth-child(1)').text().split('：')[1];
+    // 检查漫画是否已经完结
+    const finished_text = $('div.detail-list-form-title').clone().children().remove().end().text();
+    let finished = false;
+    let newOneDate = finished_text.split(',')[1];
+    if (newOneDate.includes('月') && newOneDate.includes('號')) {
+        const month = parseInt(newOneDate.split('月')[0]);
+        const date = parseInt(newOneDate.split('月')[1].split('號')[0]);
+        const year = new Date().getFullYear();
+        newOneDate = new Date(year, month - 1, date + 1);
+    } else {
+        newOneDate = new Date(newOneDate);
+        newOneDate.setDate(newOneDate.getDate() + 1);
+    }
+    if (finished_text.includes('已完結') || finished_text.includes('已完结')) {
+        finished = true;
+    }
+    // 最新一话的地址
+    const updatedOne = $('div.detail-list-form-title span.s a').attr('href');
+    const items = list
+        .map((index, item) => {
+            item = $(item);
+            const itemTitle = item.text();
+            const itemUrl = item.attr('href');
+            let itemDate;
+            if (itemUrl === updatedOne) {
+                itemDate = parseDate(newOneDate);
+            } else {
+                itemDate = '';
+            }
+            return {
+                title: itemTitle,
+                link: host + itemUrl,
+                auther: autherName,
+                pubDate: itemDate,
+                guid: host + itemUrl,
+            };
+        })
+        .get();
+    const name = $('body > div.detail-info-1 > div > div > p.detail-info-title').text();
+    let description_ = '';
+    if (finished) {
+        description_ = '已完结';
+    } else {
+        description_ = '连载中';
+    }
+    ctx.state.data = {
+        title: `x漫画  ${name}`,
+        link: `https://xmanhua.com/${uid}`,
+        description: description_,
+        allowEmpty: true,
+        item: items,
+    };
+};