rsshub 1.0.0-master.ff1cc11 → 1.0.0-master.ff97759

package/lib/v2/gzh360/universal.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { finishArticleItem } = require('./utils');
+const config = require('@/config').value;
+
+const getInitEntry = async (url) =>
+    await got(url)
+        .then((_r) => _r.data)
+        .catch((err) => {
+            if (err.response.statusCode === 404) {
+                throw new Error('This category / WeChat Official Account is not found on GZH360.');
+            }
+            throw err;
+        });
+
+module.exports = async (ctx, path, id, titleHeader = '', custom_title = null, skipAuthor = false, cacheInitEntry = false) => {
+    const rootUrl = 'http://web.gzh360.com';
+    const currentUrl = `${rootUrl}/${id ? `${path}?id=${id}` : ''}`;
+
+    const respData = cacheInitEntry ? await ctx.cache.tryGet(currentUrl, async () => await getInitEntry(currentUrl), config.cache.routeExpire, false) : await getInitEntry(currentUrl);
+    const $ = cheerio.load(respData);
+
+    const title = id ? $('head > title').text().split(' - ', 1)[0] : '首页';
+
+    let items = $('div.content div.news_desc')
+        .map((_, item) => {
+            item = $(item);
+            const link = item.find('h3 > a');
+            const href = link.attr('href');
+            const title = link.text();
+            const pubDate = item.find('span.datecss > span').attr('data-timestamp');
+            let author;
+            if (!skipAuthor) {
+                author = item.find('span.fromcss > a').text(); // only some pages have this
+            }
+            return {
+                link: `${href.startsWith('http') ? '' : rootUrl}${href}`,
+                title,
+                pubDate: parseDate(pubDate),
+                author,
+            };
+        })
+        .get();
+
+    items = await Promise.all(items.map((item) => finishArticleItem(ctx, item, skipAuthor)));
+
+    ctx.state.data = {
+        title: `${titleHeader}${custom_title ?? title}`,
+        link: currentUrl,
+        item: items.filter((item) => item),
+    };
+};

package/lib/v2/gzh360/utils.js

@@ -0,0 +1,62 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { fixArticleContent } = require('@/utils/wechat-mp');
+
+const invalidIdError = new RangeError('Invalid id');
+
+const mpIdEncode = (name) => {
+    const onePassed = Buffer.from(name).toString('base64'); // one-pass uses the standard base64 alphabet
+    // two-passed base64 is always url-safe (`7f+/` never appear), you can prove it by yourself
+    return Buffer.from(onePassed).toString('base64');
+};
+
+const mpIdDecode = (id) => {
+    // verify that the decoded name can be a valid WeChat mp name: https://kf.qq.com/faq/120911VrYVrA141110r2MRJV.html
+    // "公众号名称/昵称可设置4-30个字符（1个汉字算2字符）": 4 ASCII characters => 12, 15 Chinese characters => 80
+    // and the id is a valid two-passed base64 string (`7f+/` never appear)
+    if (id.length < 12 || id.length > 80 || id.length % 4 !== 0 || !/^[a-eg-zA-Z0-68-9]+={0,2}$/.test(id)) {
+        throw invalidIdError;
+    }
+    const deSecondPassed = Buffer.from(id, 'base64').toString();
+    // verify that it is a valid base64 string using standard base64 alphabet
+    if (deSecondPassed.length % 4 !== 0 || !/^[a-zA-Z0-9+/]+={0,2}$/.test(deSecondPassed)) {
+        throw invalidIdError;
+    }
+    const deFirstPassed = Buffer.from(deSecondPassed, 'base64').toString();
+    // "空格不可在最前或者最后，且空格不可连续"
+    if (deFirstPassed.length < 2 || deFirstPassed.startsWith(' ') || deFirstPassed.endsWith(' ') || deFirstPassed.includes('  ')) {
+        throw invalidIdError;
+    }
+    return deFirstPassed;
+};
+
+const finishArticleItem = async (ctx, item, skipAuthor = false) => {
+    // the website is slow and unstable, so we need to ignore errors to avoid breaking the whole route
+    // we can't cache the item instead of the webpage because we need to retry when the last request failed
+    const article = await ctx.cache.tryGet(
+        item.link,
+        async () =>
+            await got(item.link)
+                .then((_r) => _r.data)
+                .catch(() => null) // it is safe do that in tryGet because a false value always lead to a cache miss
+    );
+
+    if (article) {
+        const $ = cheerio.load(article);
+        if (!skipAuthor) {
+            item.author = item.author || $('[id=webbdgzh]+a').text();
+        }
+        item.title = item.title || $('div.desc > h1').text();
+        item.pubDate = item.pubDate || parseDate($('div.desc span[data-timestamp]').attr('data-timestamp'));
+        item.description = fixArticleContent($('div.rich_media_content'), true); // sometimes it is an empty string due to the website's fault
+    }
+
+    return item;
+};
+
+module.exports = {
+    mpIdEncode,
+    mpIdDecode,
+    finishArticleItem,
+};

package/lib/v2/hackernews/index.js

@@ -0,0 +1,103 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const section = ctx.params.section ?? 'index';
+    const type = ctx.params.type ?? 'sources';
+    const user = ctx.params.user ?? '';
+
+    const rootUrl = 'https://news.ycombinator.com';
+    const currentUrl = `${rootUrl}${section === 'index' ? '' : `/${section}`}${user === '' ? '' : '?id=' + user}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('.athing')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 30)
+        .map((_, thing) => {
+            thing = $(thing);
+
+            const item = {};
+
+            item.guid = thing.attr('id');
+            item.title = thing.find('.titlelink').text();
+            item.category = thing.find('.sitestr').text();
+            item.author = thing.next().find('.hnuser').text();
+            item.pubDate = parseDate(thing.find('.age').attr('title') ?? thing.next().find('.age').attr('title'));
+
+            item.link = `${rootUrl}/item?id=${item.guid}`;
+            item.origin = thing.find('.titlelink').attr('href');
+            item.onStory = thing.find('.onstory').text().substring(2);
+
+            item.comments = thing.next().find('a').last().text().split(' comment')[0];
+            item.currentComment = thing.find('.comment').text();
+            item.guid = type === 'sources' ? item.guid : `${item.guid}${item.comments === 'discuss' ? '' : `-${item.comments}`}`;
+
+            item.description = `<a href="${item.link}">Comments on Hacker News</a> | <a href="${item.origin}">Source</a>`;
+
+            return item;
+        })
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.guid, async () => {
+                if (item.comments !== 'discuss' && type === 'comments') {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+
+                    const content = cheerio.load(detailResponse.data);
+
+                    content('.reply').remove();
+
+                    item.description = '';
+
+                    content('.comtr').each(function () {
+                        const author = content(this).find('.hnuser');
+                        const comment = content(this).find('.commtext');
+
+                        item.description +=
+                            `<div><div><small><a href="${rootUrl}/${author.attr('href')}">${author.text()}</a></small>` +
+                            `&nbsp&nbsp<small><a href="${rootUrl}/item?id=${content(this).attr('id')}">` +
+                            `${content(this).find('.age').attr('title')}</a></small></div>`;
+
+                        const commentText = comment.clone();
+
+                        commentText.find('p').remove();
+                        commentText.html(`<p>${commentText.text()}</p>`);
+                        commentText.append(
+                            comment
+                                .find('p')
+                                .toArray()
+                                .map((p) => `<p>${content(p).html()}</p>`)
+                        );
+
+                        item.description += `<div>${commentText.html()}</div></div>`;
+                    });
+                } else if (item.comments !== 'discuss' && type === 'comments_list') {
+                    item.title = item.onStory;
+                    item.description = item.currentComment;
+                }
+
+                item.link = type === 'sources' ? item.origin : item.link;
+
+                delete item.origin;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/hackernews/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:section?/:type?/:user?': ['nczitzk', 'xie-dongping'],
+};

package/lib/v2/hackernews/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'ycombinator.com': {
+        _name: 'Hacker News',
+        '.': [
+            {
+                title: 'Section',
+                docs: 'https://docs.rsshub.app/programming.html#hacker-news',
+                source: ['/:section', '/'],
+                target: '/hackernews/:section?/:type?/:user?',
+            },
+        ],
+    },
+};

package/lib/v2/hackernews/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:section?/:type?/:user?', require('./index'));
+};

package/lib/v2/hameln/chapter.js

@@ -0,0 +1,59 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+    const limit = parseInt(ctx.query.limit) || 5;
+    const link = `https://syosetu.org/novel/${id}`;
+    const $ = cheerio.load(await get(link));
+
+    const title = $('span[itemprop="name"]').text();
+    const description = $('div.ss:nth-child(2)').text();
+
+    const chapter_list = $('tr[bgcolor]')
+        .map((_, chapter) => {
+            const $_chapter = $(chapter);
+            const chapter_link = $_chapter.find('a');
+            return {
+                title: chapter_link.text(),
+                link: chapter_link.attr('href'),
+                pubDate: timezone(parseDate($_chapter.find('nobr').text(), 'YYYYMMDD HH:mm'), +9),
+            };
+        })
+        .toArray()
+        .sort((a, b) => (a.pubDate <= b.pubDate ? 1 : -1))
+        .slice(0, limit);
+
+    const item_list = await Promise.all(
+        chapter_list.map((chapter) => {
+            chapter.link = `${link}/${chapter.link}`;
+            return ctx.cache.tryGet(chapter.link, async () => {
+                const content = cheerio.load(await get(chapter.link));
+                chapter.description = content('#honbun').html();
+                return chapter;
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title,
+        description,
+        link,
+        language: 'ja',
+        item: item_list,
+    };
+};
+
+const get = async (url) => {
+    const response = await got({
+        method: 'get',
+        url,
+        headers: {
+            cookie: 'over18=off',
+        },
+    });
+
+    return response.data;
+};

package/lib/v2/hameln/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/chapter/:id': ['huangliangshusheng'],
+};

package/lib/v2/hameln/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'syosetu.org': {
+        _name: 'hameln',
+        '.': [
+            {
+                title: '章节更新',
+                docs: 'https://docs.rsshub.app/reading.html#hameln-zhang-jie-geng-xin',
+                source: ['/novel/:id'],
+                target: '/hameln/chapter/:id',
+            },
+        ],
+    },
+};

package/lib/v2/hameln/router.js

@@ -0,0 +1 @@
+module.exports = (router) => router.get('/chapter/:id', require('./chapter.js'));

package/lib/v2/hdu/cs/index.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const link = 'https://computer.hdu.edu.cn';
+const host = 'https://computer.hdu.edu.cn/6738/list.htm';
+
+const getSingleRecord = async () => {
+    const res = await got({
+        method: 'get',
+        url: host,
+    });
+
+    const $ = cheerio.load(res.data);
+    const list = $('.posts-list').find('li');
+
+    return (
+        list &&
+        list
+            .map((index, item) => {
+                item = $(item);
+                const dateTxt = item.find('.date').text();
+                const date = dateTxt.slice(1, dateTxt.length - 1);
+                return {
+                    title: item.find('a').text(),
+                    pubDate: parseDate(date),
+                    link: link + item.find('a').attr('href'),
+                };
+            })
+            .get()
+    );
+};
+
+module.exports = async (ctx) => {
+    const items = await getSingleRecord();
+    const out = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                return {
+                    title: item.title,
+                    link: item.link,
+                    description: $('.wp_articlecontent').html(),
+                    pubDate: item.pubDate,
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '杭电计算机-通知公告',
+        description: '杭州电子科技大学计算机学院-通知公告',
+        link: host,
+        item: out,
+    };
+};

package/lib/v2/hdu/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/cs': ['legr4ndk'],
+};

package/lib/v2/hdu/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'hdu.edu.cn': {
+        _name: '杭州电子科技大学',
+        computer: [
+            {
+                title: '计算机学院 - 通知公告',
+                docs: 'https://docs.rsshub.app/university.html#hang-zhou-dian-zi-ke-ji-da-xue',
+                source: '/6738/list.htm',
+                target: '/hdu/cs',
+            },
+        ],
+    },
+};

package/lib/v2/hdu/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    // 杭州电子科技大学
+    router.get('/cs', require('./cs'));
+};

package/lib/v2/hellobtc/information.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const rootUrl = 'https://www.hellobtc.com';
+
+const channelSelector = {
+    latest: 'div.index_tabs_container.js-tabs-container > div:nth-child(1)',
+    application: 'div.index_tabs_container.js-tabs-container > div:nth-child(2)',
+};
+
+const titleMap = {
+    latest: '最新',
+    application: '应用',
+};
+
+module.exports = async (ctx) => {
+    const channel = ctx.params.channel ?? 'latest';
+    const url = rootUrl;
+
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const list = $(channelSelector[channel])
+        .find('div.new_item')
+        .map((_, item) => ({
+            title: $(item).find('h2').text(),
+            link: $(item).find('a').attr('href'),
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('#nr').html();
+                item.pubDate = timezone(parseDate(content('span.date').text(), 'YYYY-MM-DD HH:mm:ss'), +8);
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `白话区块链 - 首页 ${titleMap[channel]}`,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/hellobtc/kepu.js

@@ -0,0 +1,64 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const rootUrl = 'https://www.hellobtc.com';
+
+const channelSelector = {
+    latest: 'div.index_tabs_container.js-tabs-container > div:nth-child(1)',
+    bitcoin: 'div.index_tabs_container.js-tabs-container > div:nth-child(2)',
+    ethereum: 'div.index_tabs_container.js-tabs-container > div:nth-child(3)',
+    defi: 'div.index_tabs_container.js-tabs-container > div:nth-child(4)',
+    inter_blockchain: 'div.index_tabs_container.js-tabs-container > div:nth-child(5)',
+    mining: 'div.index_tabs_container.js-tabs-container > div:nth-child(6)',
+    safety: 'div.index_tabs_container.js-tabs-container > div:nth-child(7)',
+    satoshi_nakamoto: 'div.index_tabs_container.js-tabs-container > div:nth-child(8)',
+    public_blockchain: 'div.index_tabs_container.js-tabs-container > div:nth-child(9)',
+};
+
+const titleMap = {
+    latest: '最新',
+    bitcoin: '比特币',
+    ethereum: '以太坊',
+    defi: 'DeFi',
+    inter_blockchain: '跨链',
+    mining: '挖矿',
+    safety: '安全',
+    satoshi_nakamoto: '中本聪',
+    public_blockchain: '公链',
+};
+
+module.exports = async (ctx) => {
+    const channel = ctx.params.channel ?? 'latest';
+    const url = `${rootUrl}/kepu.html`;
+
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const list = $(channelSelector[channel])
+        .find('div.new_item')
+        .map((_, item) => ({
+            title: $(item).find('a').text(),
+            link: $(item).find('a').attr('href'),
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('#js_content')
+                    .html()
+                    .replace(/(<img.*?)data-src(.*?>)/g, '$1src$2');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `白话区块链 - 科普 ${titleMap[channel]}`,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/hellobtc/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/information/:channel?': ['Fatpandac'],
+    '/news': ['Fatpandac'],
+    '/kepu/:channel?': ['Fatpandac'],
+    '/topic/:id': ['Fatpandac'],
+};

package/lib/v2/hellobtc/news.js

@@ -0,0 +1,29 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const rootUrl = 'https://www.hellobtc.com';
+
+module.exports = async (ctx) => {
+    const url = `${rootUrl}/news`;
+
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const items = $('nav.js-nav')
+        .find('div.item')
+        .map((_, item) => ({
+            title: $(item).find('h2').text(),
+            link: $(item).find('a').attr('href'),
+            description: $(item).find('div.sub').text(),
+            pubDate: timezone(parseDate($(item).find('span.date').text(), 'MM-DD HH:mm'), +8),
+        }))
+        .filter((item) => item)
+        .get();
+
+    ctx.state.data = {
+        title: `白话区块链 - 快讯`,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/hellobtc/radar.js

@@ -0,0 +1,85 @@
+module.exports = {
+    'hellobtc.com': {
+        _name: '白话区块链',
+        '.': [
+            {
+                title: '首页-最新',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/'],
+                target: '/hellobtc/information/latest',
+            },
+            {
+                title: '首页-应用',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/'],
+                target: '/hellobtc/information/application',
+            },
+            {
+                title: '快讯',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/news'],
+                target: '/hellobtc/news',
+            },
+            {
+                title: '科普-最新',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/latest',
+            },
+            {
+                title: '科普-比特币',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/bitcoin',
+            },
+            {
+                title: '科普-以太坊',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/ethereum',
+            },
+            {
+                title: '科普-DeFi',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/defi',
+            },
+            {
+                title: '科普-跨链',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/inter_blockchain',
+            },
+            {
+                title: '科普-挖矿',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/mining',
+            },
+            {
+                title: '科普-安全',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/safety',
+            },
+            {
+                title: '科普-中本聪',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/satoshi_nakamoto',
+            },
+            {
+                title: '科普-公链',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/kepu.html'],
+                target: '/hellobtc/kepu/public_blockchain',
+            },
+            {
+                title: '专栏',
+                docs: 'https://docs.rsshub.app/new-media.html#bai-hua-qu-kuai-lian',
+                source: ['/topic/:id'],
+                target: (params) => `/hellobtc/topic/${params.id.split('.')[0]}`,
+            },
+        ],
+    },
+};

package/lib/v2/hellobtc/router.js

@@ -0,0 +1,6 @@
+module.exports = function (router) {
+    router.get('/information/:channel?', require('./information'));
+    router.get('/news', require('./news'));
+    router.get('/kepu/:channel?', require('./kepu'));
+    router.get('/topic/:id', require('./topic'));
+};