rsshub 1.0.0-master.fb2a407 → 1.0.0-master.fc5a2f6

package/lib/v2/hket/index.js

@@ -105,9 +105,17 @@ module.exports = async (ctx) => {
     );
 
     ctx.state.data = {
-        title: String($('head meta[name=title]').attr('content').trim()),
+        title: $('head meta[name=title]').attr('content').trim(),
         link: baseUrl + '/' + category,
-        description: String($('head meta[name=description]').attr('content').trim()),
+        description: $('head meta[name=description]').attr('content').trim(),
+        item: items,
+        language: 'zh-hk',
+    };
+
+    ctx.state.json = {
+        title: $('head meta[name=title]').attr('content').trim(),
+        link: baseUrl + '/' + category,
+        description: $('head meta[name=description]').attr('content').trim(),
         item: items,
         language: 'zh-hk',
     };

package/lib/v2/huangz/index.js

@@ -0,0 +1,29 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const BLOG_URL = 'https://blog.huangz.me/';
+
+module.exports = async (ctx) => {
+    const resp = await got({
+        method: 'get',
+        url: BLOG_URL,
+    });
+    const $ = cheerio.load(resp.body);
+    const list = $('.toctree-l2');
+
+    const items = await Promise.all(
+        list.map((index, item) => {
+            const link = `${BLOG_URL}${$(item).find('a').attr('href')}`;
+
+            return ctx.cache.tryGet(link, () => ({
+                title: $(item).find('a').text(),
+                link,
+            }));
+        })
+    );
+    ctx.state.data = {
+        title: 'huangz/blog',
+        link: 'https://blog.huangz.me',
+        item: items,
+    };
+};

package/lib/v2/huangz/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['stormbuf'],
+};

package/lib/v2/huangz/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'huangz.me': {
+        _name: '黄健宏博客',
+        blog: [
+            {
+                title: '文章',
+                docs: 'https://docs.rsshub.app/blog.html#huang-jian-hong-bo-ke',
+                source: ['/*.html'],
+                target: '/huangz',
+            },
+        ],
+    },
+};

package/lib/v2/huangz/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index'));
+};

package/lib/v2/huanqiu/index.js

@@ -0,0 +1,65 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+function getKeysRecursive(dic, key, attr, array) {
+    Object.values(dic).forEach((v) => {
+        if (v[key] !== undefined) {
+            getKeysRecursive(v[key], key, attr, array);
+        } else {
+            array.push(v[attr]);
+        }
+    });
+    return array;
+}
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'china';
+    const host = 'https://' + category + '.huanqiu.com';
+
+    const resp = await got({
+        method: 'get',
+        url: host.concat('/api/channel_pc'),
+    }).json();
+    const name = getKeysRecursive(resp.children, 'children', 'domain_name', [])[0];
+
+    const nodes = getKeysRecursive(resp.children, 'children', 'node', [])
+        .map((x) => '"' + x + '"')
+        .join();
+    const req = await got({
+        method: 'get',
+        url: host.concat('/api/list?node=') + nodes.concat(`&offset=0&limit=${ctx.query.limit ?? 20}`),
+    }).json();
+
+    let items = req.list
+        .filter((item) => item.aid)
+        .map((item) => ({
+            link: host.concat('/article/' + item.aid),
+            title: item.title,
+        }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(detailResponse.data);
+                item.description = content('article').html();
+                item.author = content('span', '.source').text();
+                item.pubDate = timezone(parseDate(content('p.time').text()), +8);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${name} - 环球网`,
+        link: host,
+        description: '环球网',
+        language: 'zh-cn',
+        item: items,
+    };
+};

package/lib/v2/huanqiu/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:category?': ['yuxinliu-alex'],
+};

package/lib/v2/huanqiu/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'huanqiu.com': {
+        _name: '环球网',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/traditional-media.html#huan-qiu',
+                source: '/',
+                target: '/news/huanqiu/:category?',
+            },
+        ],
+    },
+};

package/lib/v2/huanqiu/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/news/:category?', require('./index'));
+};

package/lib/{routes → v2}/huxiu/article.js

@@ -14,7 +14,7 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('.article-item--normal > a, .article-item--large > a')
+    const list = $('.article-item-wrap > a')
         .slice(0, 10)
         .get()
         .map((e) => $(e).attr('href'));

package/lib/{routes → v2}/huxiu/author.js

@@ -4,7 +4,8 @@ const utils = require('./utils');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const link = `https://www.huxiu.com/member/${id}.html`;
+    const link = `https://www.huxiu.com/member/${id}/article.html`;
+
     const response = await got({
         method: 'get',
         url: link,
@@ -14,12 +15,23 @@ module.exports = async (ctx) => {
     });
 
     const $ = cheerio.load(response.data);
-    const author = $('.user-name').text().trim();
+    const author = $('.author-name').text().trim();
+
+    const articleList = await got({
+        method: 'post',
+        url: 'https://account-api.huxiu.com/web/article/articleList',
+        headers: {
+            Referer: 'https://www.huxiu.com/',
+        },
+        form: {
+            platform: 'www',
+            uid: id,
+            type: 0,
+            page: 1,
+        },
+    }).json();
 
-    const list = $('.message-box > .mod-art > a')
-        .slice(0, 10)
-        .get()
-        .map((e) => $(e).attr('href'));
+    const list = articleList.data.datalist.slice(0, 10).map((item) => item.url);
 
     const items = await utils.ProcessFeed(list, ctx.cache);
 

package/lib/{routes → v2}/huxiu/collection.js

@@ -16,8 +16,7 @@ module.exports = async (ctx) => {
     const data = response.data;
     const $ = cheerio.load(data);
 
-    const list = $('.collection__article-list__item > a')
-        .filter('.article-item')
+    const list = $('.collection-lirt-item > a')
         .slice(0, 5)
         .get()
         .map((e) => $(e).attr('href'));

package/lib/v2/huxiu/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/article': ['HenryQW'],
+    '/author/:id': ['HenryQW'],
+    '/collection/:id': ['AlexdanerZe'],
+    '/tag/:id': ['xyqfer', 'HenryQW'],
+    '/search/:keyword': ['xyqfer', 'HenryQW'],
+};

package/lib/v2/huxiu/radar.js

@@ -0,0 +1,35 @@
+module.exports = {
+    'huxiu.com': {
+        _name: '虎嗅',
+        '.': [
+            {
+                title: '首页资讯',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+                source: ['/article', '/'],
+                target: '/huxiu/article',
+            },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+                source: ['/tags/:id'],
+                target: (params) => `/huxiu/tag/${params.id.replace('.html', '')}`,
+            },
+            {
+                title: '搜索',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+            },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+                source: ['/member/:id/*', '/'],
+                target: '/huxiu/author/:id',
+            },
+            {
+                title: '文集',
+                docs: 'https://docs.rsshub.app/new-media.html#hu-xiu',
+                source: ['/collection/:id', '/'],
+                target: (params) => `/huxiu/collection/${params.id.replace('.html', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/huxiu/router.js

@@ -0,0 +1,7 @@
+module.exports = (router) => {
+    router.get('/article', require('./article'));
+    router.get('/author/:id', require('./author'));
+    router.get('/collection/:id', require('./collection'));
+    router.get('/tag/:id', require('./tag'));
+    router.get('/search/:keyword', require('./search'));
+};

package/lib/{routes → v2}/huxiu/utils.js

@@ -1,17 +1,16 @@
 const cheerio = require('cheerio');
 const date = require('@/utils/date');
 const got = require('@/utils/got');
-const url = require('url');
 
 const ProcessFeed = async (list, cache) => {
     const host = 'https://www.huxiu.com';
 
     const items = await Promise.all(
         list.map(async (e) => {
-            const link = url.resolve(host, e);
+            const link = new URL(e, host).href;
 
             const single = await cache.tryGet(link, async () => {
-                const response = await got.get(link);
+                const response = await got(link);
 
                 const $ = cheerio.load(response.data);
 
@@ -58,7 +57,7 @@ const ProcessFeed = async (list, cache) => {
                 };
             });
 
-            return Promise.resolve(single);
+            return single;
         })
     );
 

package/lib/v2/ieee/templates/description.art

@@ -1,5 +1,7 @@
 <p>
     <span><big>{{ item.title }}</big></span><br />
+</p>
+<p>
     <span><small><i>{{ item.authors }}</i></small></span><br />
     <span><small><i>https://doi.org/{{ item.doi }}</i></small></span><br />
     <span><small><i>Volume: {{ item.volume }}</i></small></span><br />

package/lib/v2/ielts/index.js

@@ -3,37 +3,38 @@ const got = require('@/utils/got');
 const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 const targetUrl = 'https://ielts.neea.cn/allnews?locale=zh_CN';
+const config = require('@/config').value;
 
 module.exports = async (ctx) => {
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
-    await page.evaluateOnNewDocument(() => {
-        const proto = navigator.__proto__;
-        delete proto.webdriver;
-    });
-
-    await page.goto(targetUrl, {
-        waitUntil: 'networkidle0',
-    });
-    const list = await page.evaluate(() =>
-        window
-            .$('#newsListUl li')
-            .get()
-            .map((elem) => {
-                const $elem = window.$(elem);
-                return {
-                    title: $elem.find('a').text(),
-                    link: $elem.find('a').attr('href'),
-                    pubDate: $elem.find('span').eq(-1).text().replace(/[[\]]/g, '').trim(),
-                };
-            })
+    const html = await ctx.cache.tryGet(
+        targetUrl,
+        async () => {
+            const browser = await require('@/utils/puppeteer')({ stealth: true });
+            const page = await browser.newPage();
+
+            await page.goto(targetUrl);
+            await page.waitForSelector('div.container');
+
+            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            browser.close();
+            return html;
+        },
+        config.cache.routeExpire,
+        false
     );
-    browser.close();
 
-    list.sort((a, b) => parseDate(b.pubDate) - parseDate(a.pubDate));
-    list.forEach((item) => {
-        item.pubDate = timezone(parseDate(item.pubDate), +8);
-    });
+    const $ = cheerio.load(html);
+
+    const list = $('#newsListUl li')
+        .get()
+        .map((elem) => {
+            const $elem = $(elem);
+            return {
+                title: $elem.find('a').text(),
+                link: $elem.find('a').attr('href'),
+                pubDate: timezone(parseDate($elem.find('span').eq(-1).text().replace(/[[\]]/g, '').trim(), +8)),
+            };
+        });
 
     const items = await Promise.all(
         list.map((item) =>

package/lib/v2/iguoguo/index.js

@@ -0,0 +1,75 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const rootUrl = 'https://www.iguoguo.net';
+
+const getChildrenFromCheerioRoot = (cheerioInstance) => {
+    const root = cheerioInstance.root();
+    const rootKeys = Object.keys(root).filter((n) => n % 1 === 0);
+    const rootChildren = rootKeys.map((key) => root[key].children);
+    return rootChildren[0];
+};
+
+const getComments = (a, node) => {
+    if (node.type === 'comment') {
+        return a.concat([node.data]);
+    } else if (node.type === 'tag' && node.children && node.children.length > 0) {
+        return a.concat(node.children.reduce(getComments, []));
+    } else {
+        return a;
+    }
+};
+
+module.exports = async (ctx) => {
+    const currentUrl = rootUrl.concat('/html5');
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+    const $ = cheerio.load(response.data);
+    const list = $('a', '.post')
+        .filter((_, x) => $(x).attr('href').endsWith('.html'))
+        .filter((_, x) => $(x).children().eq(0).attr('src'))
+        .map((_, item) => ({
+            link: $(item).attr('href'),
+            cover: $(item).children().eq(0).attr('src'),
+        }))
+        .get();
+
+    const mime = {
+        jpg: 'jpeg',
+        png: 'png',
+    };
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(detailResponse.data);
+                item.title = content('h1', '.post-info').text();
+                item.description = content('div.clearfix', '.post_content').html();
+                const comments = getChildrenFromCheerioRoot(content).reduce(getComments, []);
+                const date = comments[comments.length - 1].trim().split(' ').slice(9, 11);
+                item.pubDate = timezone(parseDate(date[0] + date[1], 'YYYY-MM-DDHH:mm:ss'), +8);
+                item.media = {
+                    content: {
+                        url: item.cover,
+                        type: `image/${mime[item.cover.split('.').pop()]}`,
+                    },
+                };
+                return item;
+            })
+        )
+    );
+    ctx.state.data = {
+        title: '爱果果',
+        link: currentUrl,
+        description: '爱果果iguoguo是一个优秀酷站、h5、UI素材资源的发布分享平台，是设计师的灵感聚合地和素材下载源。',
+        language: 'zh-cn',
+        item: items,
+    };
+};

package/lib/v2/iguoguo/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/html5': ['yuxinliu-alex'],
+};

package/lib/v2/iguoguo/radar.js

@@ -0,0 +1,11 @@
+module.exports = {
+    'iguoguo.net': {
+        _name: '爱果果',
+        '.': [
+            {
+                title: '最新H5',
+                docs: 'https://docs.rsshub.app/design.html#ai-guo-guo-h5',
+            },
+        ],
+    },
+};

package/lib/v2/iguoguo/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/html5', require('./index'));
+};

package/lib/v2/itch/devlog.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const user = ctx.params.user ?? '';
+    const id = ctx.params.id ?? '';
+
+    const rootUrl = `https://${user}.itch.io/${id}/devlog`;
+
+    const response = await got({
+        method: 'get',
+        url: rootUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.title')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+                pubDate: timezone(parseDate(item.text()), +8),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.author = detailResponse.data.match(/"author":{".*?","name":"(.*?)"/)[1];
+                item.pubDate = parseDate(detailResponse.data.match(/"datePublished":"(.*?)"/)[1]);
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    images: content('.post_image')
+                        .toArray()
+                        .map((i) => content(i).attr('src')),
+                    description: content('.post_body').html(),
+                });
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/itch/index.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://itch.io';
+    const currentUrl = `${rootUrl}${ctx.path}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.title.game_link')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.author = content('title').text().split('by ').pop();
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    images: content('.screenshot')
+                        .toArray()
+                        .map((i) => content(i).attr('src')),
+                    description: content('.formatted_description').html(),
+                });
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/itch/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/devlog/:user/:id': ['nczitzk'],
+    '/posts/:topic/:id': ['nczitzk'],
+    '/:sort?/:tag?': ['nczitzk'],
+};

package/lib/v2/itch/posts.js

@@ -0,0 +1,41 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const topic = ctx.params.topic;
+    const id = ctx.params.id;
+
+    const rootUrl = 'https://itch.io';
+    const currentUrl = `${rootUrl}/t/${topic}/${id}?before=999999999`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const items = $('.post_grid')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const author = item.find('.post_author').text();
+            const description = item.find('.post_body');
+
+            return {
+                author,
+                description: description.html(),
+                title: `${author}: ${description.text()}`,
+                link: item.find('.post_date a').attr('href'),
+                pubDate: parseDate(item.find('.post_date').attr('title')),
+            };
+        });
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};