rsshub 1.0.0-master.f8b1c8d → 1.0.0-master.f91e29d

package/lib/v2/wnacg/index.js

@@ -0,0 +1,110 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const categories = {
+    1: '同人誌 漢化',
+    2: '同人誌 CG畫集',
+    3: '同人誌 Cosplay',
+    5: '同人誌',
+    6: '單行本',
+    7: '雜誌&短篇',
+    9: '單行本 漢化',
+    10: '雜誌&短篇 漢化',
+    12: '同人誌 日語',
+    13: '單行本 日語',
+    14: '雜誌&短篇 日語',
+    16: '同人誌 English',
+    17: '單行本 English',
+    19: '韓漫',
+    20: '韓漫 漢化',
+    21: '韓漫 生肉',
+};
+
+const baseUrl = 'https://wnacg.org';
+
+module.exports = async (ctx) => {
+    const { cid, tag } = ctx.params;
+    if (cid && !Object.keys(categories).includes(cid)) {
+        throw Error('此分类不存在');
+    }
+
+    const url = `${baseUrl}/albums${cid ? `-index-cate-${cid}` : ''}${tag ? `-index-tag-${tag}` : ''}.html`;
+    const { data } = await got(url);
+    const $ = cheerio.load(data);
+
+    const list = $('.gallary_item')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const href = item.find('a').attr('href');
+            const aid = href.match(/^\/photos-index-aid-(\d+)\.html$/)[1];
+            return {
+                title: item.find('a').attr('title'),
+                link: `${baseUrl}${href}`,
+                pubDate: parseDate(
+                    item
+                        .find('.info_col')
+                        .text()
+                        .replace(/\d+張照片，\n創建於/, ''),
+                    'YYYY-MM-DD'
+                ),
+                aid,
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: descRes } = await got(item.link, {
+                    headers: {
+                        referer: encodeURI(url),
+                    },
+                });
+                let $ = cheerio.load(descRes);
+                const author = $('.uwuinfo p').first().text();
+                const category = $('.tagshow')
+                    .toArray()
+                    .map((item) => $(item).text());
+                $('.addtags').remove();
+                const description = $('.uwconn').html();
+
+                const { data } = await got(`${baseUrl}/photos-gallery-aid-${item.aid}.html`, {
+                    headers: {
+                        referer: `${baseUrl}/photos-slide-aid-${item.aid}.html`,
+                    },
+                });
+                $ = cheerio.load(data);
+
+                const imgListMatch = $('script')
+                    .text()
+                    .match(/var imglist = (\[.*\]);"\);/)[1];
+
+                const imgList = JSON.parse(
+                    imgListMatch
+                        .replace(/url:/g, '"url":')
+                        .replace(/caption:/g, '"caption":')
+                        .replace(/fast_img_host\+\\/g, '')
+                        .replace(/\\/g, '')
+                );
+
+                item.author = author;
+                item.category = category;
+                item.description = art(path.join(__dirname, 'templates/manga.art'), {
+                    description,
+                    imgList,
+                });
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/wnacg/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/': ['KenMizz'],
+    '/category/:cid': ['Gandum2077'],
+    '/tag/:tag': ['Gandum2077'],
+};

package/lib/v2/wnacg/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'wnacg.org': {
+        _name: '紳士漫畫',
+        '.': [
+            {
+                title: '最新',
+                docs: 'https://docs.rsshub.app/anime.html#shen-shi-man-hua',
+                source: ['/albums.html', '/'],
+                target: '/wnacg',
+            },
+            {
+                title: '分类更新',
+                docs: 'https://docs.rsshub.app/anime.html#shen-shi-man-hua',
+                source: ['/*'],
+                target: (_, url) => `/wnacg/category/${new URL(url).pathname.match(/albums-index-cate-(\d+)\.html$/)[1]}`,
+            },
+            {
+                title: '標籤更新',
+                docs: 'https://docs.rsshub.app/anime.html#shen-shi-man-hua',
+                source: ['/*'],
+                target: (_, url) => `/wnacg/tag/${new URL(url).pathname.match(/albums-index-tag-(.+?)\.html$/)[1]}`,
+            },
+        ],
+    },
+};

package/lib/v2/wnacg/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/', require('./index'));
+    router.get('/category/:cid', require('./index'));
+    router.get('/tag/:tag', require('./index'));
+};

package/lib/v2/wnacg/templates/manga.art

@@ -0,0 +1,9 @@
+{{ if description }}
+{{@ description }}
+{{ /if }}
+<br>
+{{ if imgList }}
+{{ each imgList img }}
+<img src="https:{{ img.url }}" alt="{{ img.caption }}">
+{{ /each }}
+{{ /if }}

package/lib/v2/woshipm/bookmarks.js

@@ -0,0 +1,34 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+    const { data: profile } = await got(`${baseUrl}/api2/user/profile`, {
+        searchParams: {
+            uid: id,
+        },
+    });
+    const userInfo = profile.RESULT.userInfoVo;
+    const name = userInfo.nickName;
+
+    const remark_api = `${baseUrl}/__api/v1/users/${id}/bookmarks`;
+    const response_api = await got(remark_api);
+    const list = response_api.data.payload.value.map((item) => ({
+        title: item.title,
+        description: item.snipper,
+        link: item.permalink,
+        pubDate: parseDate(item.date, 'YYYY/MM/DD'),
+        author: item.author.name,
+    }));
+
+    const out = await Promise.all(list.map((info) => parseArticle(info, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `${name}的收藏-人人都是产品经理`,
+        description: userInfo.description,
+        image: userInfo.avartar,
+        link: `${baseUrl}/u/${id}`,
+        item: out,
+    };
+};

package/lib/v2/woshipm/latest.js

@@ -0,0 +1,27 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const pages = 3;
+    const urlList = Array.from({ length: pages }, (_, v) => `${baseUrl}/__api/v1/stream-list${v < 1 ? '' : `/page/${v + 1}`}`);
+    const responses = await got.all(urlList.map((url) => got(url)));
+
+    const payload = responses.map((response) => response.data.payload).flat();
+
+    const list = payload.map((item) => ({
+        title: item.title,
+        description: item.snipper,
+        link: item.permalink,
+        pubDate: parseDate(item.date, 'YYYY/MM/DD'),
+        author: item.author.name,
+    }));
+
+    const result = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: '最新文章 - 人人都是产品经理',
+        link: baseUrl,
+        item: result,
+    };
+};

package/lib/v2/woshipm/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/bookmarks/:id': ['LogicJake'],
+    '/latest': ['Director-0428'],
+    '/popular/:range?': ['WenryXu'],
+    '/user_article/:id': ['LogicJake'],
+    '/wen': ['WenryXu'],
+};

package/lib/v2/woshipm/popular.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, parseArticle } = require('./utils');
+
+const rangeMap = {
+    daily: '日榜',
+    weekly: '周榜',
+    monthly: '月榜',
+};
+
+module.exports = async (ctx) => {
+    const { range = 'daily' } = ctx.params;
+    const { data: response } = await got(`${baseUrl}/api2/app/article/popular/${range}`);
+
+    const list = response.RESULT.map((item) => {
+        item = item.data;
+        return {
+            title: item.articleTitle,
+            description: item.articleSummary,
+            link: `${baseUrl}/${item.type}/${item.id}.html`,
+            pubDate: parseDate(item.publishTime, 'x'),
+            author: item.articleAuthor,
+        };
+    });
+
+    const result = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `热门文章 - ${rangeMap[range]} - 人人都是产品经理`,
+        link: baseUrl,
+        item: result,
+    };
+};

package/lib/v2/woshipm/radar.js

@@ -0,0 +1,39 @@
+module.exports = {
+    'woshipm.com': {
+        _name: '人人都是产品经理',
+        '.': [
+            {
+                title: '热门文章',
+                docs: 'https://docs.rsshub.app/new-media.html#ren-ren-dui-shi-chan-pin-jing-li',
+                source: ['/'],
+                target: '/woshipm/popular',
+            },
+            {
+                title: '用户收藏',
+                docs: 'https://docs.rsshub.app/new-media.html#ren-ren-dui-shi-chan-pin-jing-li',
+                source: ['/u/:id'],
+                target: '/woshipm/bookmarks/:id',
+            },
+            {
+                title: '用户文章',
+                docs: 'https://docs.rsshub.app/new-media.html#ren-ren-dui-shi-chan-pin-jing-li',
+                source: ['/u/:id'],
+                target: '/woshipm/user_article/:id',
+            },
+            {
+                title: '最新文章',
+                docs: 'https://docs.rsshub.app/new-media.html#ren-ren-dui-shi-chan-pin-jing-li',
+                source: ['/'],
+                target: '/woshipm/latest',
+            },
+        ],
+        wen: [
+            {
+                title: '天天问',
+                docs: 'https://docs.rsshub.app/new-media.html#ren-ren-dui-shi-chan-pin-jing-li',
+                source: ['/'],
+                target: '/woshipm/wen',
+            },
+        ],
+    },
+};

package/lib/v2/woshipm/router.js

@@ -0,0 +1,7 @@
+module.exports = (router) => {
+    router.get('/bookmarks/:id', require('./bookmarks'));
+    router.get('/latest', require('./latest'));
+    router.get('/popular/:range?', require('./popular'));
+    router.get('/user_article/:id', require('./user_article'));
+    router.get('/wen', require('./wen'));
+};

package/lib/v2/woshipm/user_article.js

@@ -0,0 +1,37 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { baseUrl, parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 12;
+    const link = `${baseUrl}/u/${id}`;
+
+    const data = await got(link).then((res) => res.data);
+    const $ = cheerio.load(data);
+    const name = $('.author--meta .name').text();
+
+    const list = $('.post--card')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const postCardTitle = item.find('h2.post--card__title a');
+            return {
+                title: postCardTitle.attr('title'),
+                link: postCardTitle.attr('href'),
+                pubDate: parseDate(item.find('time').text(), 'YYYY-MM-DD'),
+            };
+        });
+
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `${name}的文章-人人都是产品经理`,
+        description: $('.author--meta .description').text(),
+        image: $('.author--meta .avatar').attr('src').split('!')[0],
+        link,
+        item: items,
+    };
+};

package/lib/v2/woshipm/utils.js

@@ -0,0 +1,21 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const baseUrl = 'https://www.woshipm.com';
+
+const parseArticle = (item, tryGet) =>
+    tryGet(item.link, async () => {
+        const { data: response } = await got(item.link);
+
+        const $ = cheerio.load(response);
+        $('.support-author').remove();
+
+        item.description = $('.article--content').html();
+
+        return item;
+    });
+
+module.exports = {
+    baseUrl,
+    parseArticle,
+};

package/lib/v2/woshipm/wen.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const baseUrl = 'https://wen.woshipm.com';
+const { parseRelativeDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const response = await got(`${baseUrl}/m/main/indexNewData.html`);
+    const $ = cheerio.load(response.data);
+    const postList = $('.article-list-item')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.went-head-text').text(),
+                link: `${baseUrl}${item.find('.went-head').attr('href')}`,
+                pubDate: parseRelativeDate(item.find('.list-text').text().split('|')[1]),
+            };
+        });
+
+    const result = await Promise.all(
+        postList.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const temp = await got(item.link);
+                const $ = cheerio.load(temp.data);
+                item.description = $('.wt-desc').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '天天问 - 人人都是产品经理',
+        link: baseUrl,
+        item: result,
+    };
+};

package/lib/v2/wp-china/news.js

@@ -33,19 +33,18 @@ module.exports = async (ctx) => {
         .get();
 
     const items = await Promise.all(
-        list.map(
-            async (item) =>
-                await ctx.cache.tryGet(item.link, async () => {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                    });
-                    const content = cheerio.load(detailResponse.data);
-
-                    item.description = content('.wb100').html();
-
-                    return item;
-                })
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('.wb100').html();
+
+                return item;
+            })
         )
     );
 

package/lib/v2/wsj/news.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { asyncPoolAll, parseArticle } = require('./utils');
+const hostMap = {
+    'en-us': 'https://www.wsj.com',
+    'zh-cn': 'https://cn.wsj.com/zh-hans',
+    'zh-tw': 'https://cn.wsj.com/zh-hant',
+};
+module.exports = async (ctx) => {
+    const lang = ctx.params.lang;
+    const category = ctx.params.category || '';
+    const host = hostMap[lang];
+    let subTitle = ` - ${lang.toUpperCase()}`;
+    let url = host;
+    if (category.length > 0) {
+        url = `${host}/news/${category}`;
+        subTitle = `${subTitle} - ${category}`;
+    }
+    const response = await got({
+        method: 'get',
+        url,
+    });
+
+    const $ = cheerio.load(response.data);
+    const contents = $('script:contains("window.__STATE__")').text();
+    const data = JSON.parse(contents.match(/\{.*\}/)[0]).data;
+    const filteredKeys = Object.entries(data)
+        .filter(([key, value]) => {
+            if (!key.startsWith('article')) {
+                return false;
+            }
+            const link = value.data.data.url;
+            return link.includes('wsj.com/articles/');
+        })
+        .map(([key]) => key);
+    const list = filteredKeys.map((key) => {
+        const item = {};
+        item.title = data[key].data.data.headline;
+        item.link = data[key].data.data.url;
+        item.test = key;
+        return item;
+    });
+    const items = await asyncPoolAll(10, list, (item) => parseArticle(item, ctx));
+
+    ctx.state.data = {
+        title: `WSJ${subTitle}`,
+        link: url,
+        description: `WSJ${subTitle}`,
+        item: items,
+    };
+};

package/lib/v2/wsj/radar.js

@@ -5,7 +5,7 @@ module.exports = {
             {
                 title: '新闻',
                 docs: 'https://docs.rsshub.app/traditional-media.html#hua-er-jie-ri-bao-the-wall-street-journal-wsj',
-                souce: '/',
+                source: '/',
                 target: '/wsj/zh-cn',
             },
         ],
@@ -13,7 +13,7 @@ module.exports = {
             {
                 title: '新闻',
                 docs: 'https://docs.rsshub.app/traditional-media.html#hua-er-jie-ri-bao-the-wall-street-journal-wsj',
-                souce: '/',
+                source: '/',
                 target: '/wsj/en-us',
             },
         ],

package/lib/v2/wsj/router.js

@@ -1,3 +1,3 @@
 module.exports = (router) => {
-    router.get('/:lang/:category?', require('./index'));
+    router.get('/:lang/:category?', require('./news'));
 };

package/lib/v2/wsj/templates/article-description.art

@@ -0,0 +1,6 @@
+<div>
+    {{@ item.subTitle}}
+    {{@ item.article}}
+
+</div>
+

package/lib/v2/wsj/utils.js

@@ -0,0 +1,121 @@
+const asyncPool = require('tiny-async-pool');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const UA = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
+
+// const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
+const parseArticle = (item, ctx) =>
+    ctx.cache.tryGet(item.link, async () => {
+        // Fetch the AMP version
+        const url = item.link.replace(/(?<=^https:\/\/\w+\.wsj\.com)/, '/amp');
+        const response = await got({
+            url,
+            method: 'get',
+            headers: {
+                'User-Agent': UA,
+            },
+        });
+        const html = response.data;
+        const $ = cheerio.load(html);
+
+        // Summary
+        const summary = $('head > meta[name="description"]').attr('content');
+
+        // Metadata (categories & updatedAt)
+        const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
+        const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
+        const author = $('.author > a[rel="author"]').text();
+
+        const categories = $('meta[name="keywords"]')
+            .attr('content')
+            .split(',')
+            .map((c) => c.trim());
+
+        const article = $('article');
+        item.subTitle = $('h2.sub-head').html();
+
+        // Remove podcast
+        article.find('.media-object-podcast').remove();
+
+        // Authors
+        article.find('.bylineWrap').each((i, e) => {
+            $(e)
+                .find('p')
+                .each(function () {
+                    $(this).replaceWith($(this).html());
+                });
+        });
+
+        // Images
+        article.find('.bigTop-hero').each((i, e) => {
+            // console.log($(e).html());
+            const imgSrc = $(e).find('amp-img').attr('src');
+            const imgAlt = $(e).find('amp-img').attr('alt');
+            const figCaption = $(e).find('.imageCaption').text().trim();
+            const figCredit = $(e).find('.imageCredit').text().trim();
+            const fig = $(`<figure><img src="${imgSrc}" alt="${imgAlt}"><figcaption>${figCaption} <span>${figCredit}</span></figcaption></figure>`);
+            $(fig).insertBefore(e);
+            $(e).remove();
+        });
+        article.find('amp-img').each((i, e) => {
+            const img = $(`<img width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}" alt="${e.attribs.alt}">`);
+
+            // Caption follows, no need to handle caption
+            $(img).insertBefore(e);
+            $(e).remove();
+        });
+
+        // iframes (youtube videos and interactive elements)
+        article.find('amp-iframe').each((i, e) => {
+            const iframe = $(`<iframe width="${e.attribs.width}" height="${e.attribs.height}" src="${e.attribs.src}">`);
+            $(iframe).insertBefore(e);
+            $(e).remove();
+        });
+
+        // Remove unwanted DOMs
+        const unwanted_element_selectors = ['amp-ad', '.wsj-ad', 'h1', 'h2', '.zonedModule', '.snippet', '.newsletter-inset'];
+        unwanted_element_selectors.forEach((selector) => {
+            article.find(selector).each((i, e) => {
+                $(e).remove();
+            });
+        });
+
+        // Paywall
+        article.find('.paywall').each((i, e) => {
+            // Caption follows, no need to handle caption
+            $(e.childNodes).insertBefore(e);
+            $(e).remove();
+        });
+        item.article = article.html();
+        item.description = art(path.join(__dirname, 'templates/article-description.art'), {
+            item,
+        });
+
+        return {
+            title: item.title,
+            pubDate: parseDate(publishedAt),
+            updated: parseDate(updatedAt),
+            author,
+            link: item.link,
+            summary,
+            description: item.description,
+            category: categories,
+            icon: 'https://s.wsj.net/media/wsj_launcher-icon-4x.png',
+            logo: 'https://vir.wsj.net/fp/assets/webpack4/img/wsj-logo-big-black.165e51cc.svg',
+        };
+    });
+
+const asyncPoolAll = async (...args) => {
+    const results = [];
+    for await (const result of asyncPool(...args)) {
+        results.push(result);
+    }
+    return results;
+};
+module.exports = {
+    asyncPoolAll,
+    parseArticle,
+};

package/lib/v2/wxkol/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/show/:id': ['TonyRL'],
+};

package/lib/v2/wxkol/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'wxkol.com': {
+        _name: '微小领',
+        '.': [
+            {
+                title: '微信公众号',
+                docs: 'https://docs.rsshub.app/new-media.html#wei-xiao-ling',
+                source: ['/show/:id'],
+                target: (params) => `/wxkol/show/${params.id.replace('.html', '')}`,
+            },
+        ],
+    },
+};