rsshub 1.0.0-master.fe50f9c → 1.0.0-master.fe6117a

package/lib/v2/gocn/util.js

@@ -1,17 +0,0 @@
-const got = require('@/utils/got');
-const { parseDate } = require('@/utils/parse-date');
-
-async function getFeedItem(item) {
-    const response = await got(`https://gocn.vip/apiv3/topic/${item.guid}/info`);
-
-    return {
-        link: `https://gocn.vip/topics/${item.guid}`,
-        title: item.title,
-        description: response.data.data.topic.contentHtml,
-        pubDate: parseDate(item.ctime, 'X'),
-    };
-}
-
-module.exports = {
-    getFeedItem,
-};

package/lib/v2/google/sites.js

@@ -1,75 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-const base = 'https://sites.google.com/';
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id;
-
-    const init_url = `https://sites.google.com/site/${id}/system/app/pages/sitemap/list?offset=0`;
-    const init_response = await got.get(init_url);
-
-    const test_pages_count = init_response.data.match(/'sites-pagination-next-link-top', (\d+),/);
-    if (!test_pages_count) {
-        throw 'Site Not Found';
-    }
-    const pages_count = parseInt(test_pages_count[1]);
-
-    const url = pages_count > 20 ? `https://sites.google.com/site/${id}/system/app/pages/sitemap/list?offset=${pages_count - 20}` : init_url;
-    const response = url === init_url ? init_response : await got.get(url);
-
-    const $ = cheerio.load(response.data);
-
-    const site_name = $('a#sites-chrome-userheader-title').text();
-    const list = $('.sites-table > tbody > tr').get();
-
-    const parseContent = (htmlString) => {
-        const $ = cheerio.load(htmlString);
-
-        const content = $('#sites-canvas-main-content');
-
-        return {
-            description: content.html(),
-        };
-    };
-
-    const out = await Promise.all(
-        list.map(async (item, index) => {
-            const $ = cheerio.load(item);
-
-            const title = $('a');
-            const path = title.attr('href');
-            const link = base + path;
-            const time = new Date();
-            time.setMinutes(time.getMinutes() - pages_count + index);
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const rssitem = {
-                title: title.text().trim(),
-                link,
-                pubDate: time,
-                author: site_name,
-            };
-
-            try {
-                const response = await got.get(link);
-                const result = parseContent(response.data);
-                rssitem.description = result.description;
-            } catch (err) {
-                return Promise.resolve('');
-            }
-            ctx.cache.set(link, JSON.stringify(rssitem));
-            return Promise.resolve(rssitem);
-        })
-    );
-
-    ctx.state.data = {
-        title: `${site_name} - Google Sites`,
-        link: url,
-        item: out.filter((item) => item !== ''),
-    };
-};

package/lib/v2/google/sitesRecentChanges.js

@@ -1,52 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id;
-
-    const rootUrl = 'https://sites.google.com';
-    const currentUrl = `${rootUrl}/site/${id}/activity.xml`;
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-
-    const $ = cheerio.load(response.data);
-
-    let items = $('entry')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-
-            return {
-                title: item.find('summary').text(),
-                link: item.find('summary a').attr('href'),
-                pubDate: parseDate(item.find('updated').text()),
-                author: item.find('author name').text(),
-            };
-        });
-
-    items = await Promise.all(
-        items.map(
-            async (item) =>
-                await ctx.cache.tryGet(item.link, async () => {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                    });
-                    const content = cheerio.load(detailResponse.data);
-
-                    item.description = content('#sites-canvas').html();
-
-                    return item;
-                })
-        )
-    );
-
-    ctx.state.data = {
-        title: $('feed title').first().text(),
-        link: currentUrl,
-        item: items,
-    };
-};

package/lib/v2/gov/sichuan/deyang/govpulicinfo.js

@@ -1,95 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const { art } = require('@/utils/render');
-const path = require('path');
-
-// 各地区url信息
-const basicInfoDict = {
-    绵竹市: {
-        rootUrl: 'https://www.mz.gov.cn',
-        infoUrl: 'https://www.mz.gov.cn/info/iList.jsp?tm_id=1805&cur_page=',
-    },
-};
-
-const getInfoUrlList = async (countyName) => {
-    const url = basicInfoDict[countyName].infoUrl;
-    const infoUrlList = [];
-    const expectToday = new Date();
-    for (let pageNum = 1; ; pageNum++) {
-        let response;
-        try {
-            // eslint-disable-next-line no-await-in-loop
-            response = await got(`${url}${pageNum}`);
-        } catch (err) {
-            // eslint-disable-next-line no-await-in-loop
-            response = await got(`${url}${pageNum}`, { dnsLookupIpVersion: 'ipv6' });
-        }
-        const $ = cheerio.load(response.data);
-        const infoList = $('#list_content > ul > li');
-        // 判断当前页是否有数据
-        if (infoList.length !== 0) {
-            for (let infoIdx = 0; infoIdx < infoList.length; infoIdx++) {
-                const date = parseDate($('span', infoList[infoIdx]).html());
-                // 判断当前信息日期是否是今日
-                if (date.getUTCFullYear() === expectToday.getUTCFullYear() && date.getUTCMonth() === expectToday.getUTCMonth() && date.getUTCDate() === expectToday.getUTCDate() - 1) {
-                    infoUrlList.push(`${basicInfoDict[countyName].rootUrl}${$('a', infoList[infoIdx]).attr('href')}`);
-                } else {
-                    return infoUrlList;
-                }
-            }
-        } else {
-            return infoUrlList;
-        }
-    }
-};
-
-// 获取信息正文内容
-const getInfoContent = async (ctx, url, countyName) =>
-    await ctx.cache.tryGet(url, async () => {
-        let response;
-        try {
-            response = await got(url);
-        } catch (err) {
-            response = await got(url, { dnsLookupIpVersion: 'ipv6' });
-        }
-        const $ = cheerio.load(response.data);
-        const fileNodes = $('#symbol > div:nth-child(4) > div > span > a');
-        const fileList = [];
-        for (let i = 0; i < fileNodes.length; i++) {
-            fileList.push({
-                name: $(fileNodes[i]).text(),
-                url: `${basicInfoDict[countyName].rootUrl}/${$(fileNodes[i]).attr('href')}`,
-            });
-        }
-        const rawDate = $('#symbol > div:nth-child(1) > div:nth-child(3)').text().split('：')[1].trim();
-        return {
-            title: $('#main').text().trim(),
-            id: $('#symbol > div:nth-child(1) > div:nth-child(1)').text().split('：')[1].trim(),
-            infoNum: $('#symbol > div:nth-child(1) > div:nth-child(2) > span').text().split('：')[1].trim(),
-            pubDate: parseDate(rawDate),
-            date: rawDate,
-            keyWord: $('#symbol > div:nth-child(2) > div:nth-child(3)').text().split('：')[1].trim(),
-            source: $('#symbol > div:nth-child(2) > div:nth-child(2)').text().split('：')[1].trim(),
-            content: $('#container > div.ewb-white > div.ewb-article-detail').html(),
-            file: fileList,
-            link: url,
-        };
-    });
-
-module.exports = async (ctx) => {
-    const countyName = ctx.params.countyName;
-    const infoUrlList = await getInfoUrlList(countyName);
-    const items = await Promise.all(infoUrlList.map(async (item) => await getInfoContent(ctx, item, countyName)));
-
-    ctx.state.data = {
-        title: `政府公开信息 - ${countyName}`,
-        link: basicInfoDict[countyName].infoUrl,
-        item: items.map((item) => ({
-            title: item.title,
-            description: art(path.resolve(__dirname, './templates/govPublicInfo.art'), { item }),
-            link: item.link,
-            pubDate: item.pubDate,
-        })),
-    };
-};

package/lib/v2/gzh360/_README

@@ -1 +0,0 @@
-Make sure you read lib/utils/wechat-mp.js before adding a new route.

package/lib/v2/gzh360/category.js

@@ -1,8 +0,0 @@
-const universal = require('./universal');
-
-module.exports = async (ctx) => {
-    const path = 'category';
-    const id = ctx.params.id ?? '';
-    const titleHeader = '公众号 360 - ';
-    await universal(ctx, path, id, titleHeader);
-};

package/lib/v2/gzh360/gzh.js

@@ -1,18 +0,0 @@
-const { mpIdEncode, mpIdDecode } = require('./utils');
-const universal = require('./universal');
-
-module.exports = async (ctx) => {
-    const path = 'gzh_articles';
-
-    let name = ctx.params.name;
-    let id = name;
-
-    // in order to support RSSHub Radar, we need to accept both name and id
-    try {
-        name = mpIdDecode(name);
-    } catch {
-        id = mpIdEncode(name);
-    }
-
-    await universal(ctx, path, id, '', name, true, true);
-};

package/lib/v2/gzh360/maintainer.js

@@ -1,4 +0,0 @@
-module.exports = {
-    '/category/:id?': ['Rongronggg9'],
-    '/gzh/:name?': ['Rongronggg9'],
-};

package/lib/v2/gzh360/radar.js

@@ -1,19 +0,0 @@
-module.exports = {
-    'gzh360.com': {
-        _name: '公众号360',
-        web: [
-            {
-                title: '公众号',
-                docs: 'https://docs.rsshub.app/new-media.html#gong-zhong-hao-360',
-                source: ['/gzh_articles', '/gzh', '/'],
-                target: (params, url) => `/gzh360/gzh/${new URL(url).searchParams.get('id') ?? ''}`,
-            },
-            {
-                title: '分类',
-                docs: 'https://docs.rsshub.app/new-media.html#gong-zhong-hao-360',
-                source: ['/category', '/'],
-                target: (params, url) => `/gzh360/category/${new URL(url).searchParams.get('id') ?? ''}`,
-            },
-        ],
-    },
-};

package/lib/v2/gzh360/router.js

@@ -1,4 +0,0 @@
-module.exports = function (router) {
-    router.get('/category/:id?', require('./category'));
-    router.get('/gzh/:name', require('./gzh'));
-};

package/lib/v2/gzh360/universal.js

@@ -1,53 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const { finishArticleItem } = require('./utils');
-const config = require('@/config').value;
-
-const getInitEntry = async (url) =>
-    await got(url)
-        .then((_r) => _r.data)
-        .catch((err) => {
-            if (err.response.statusCode === 404) {
-                throw new Error('This category / WeChat Official Account is not found on GZH360.');
-            }
-            throw err;
-        });
-
-module.exports = async (ctx, path, id, titleHeader = '', custom_title = null, skipAuthor = false, cacheInitEntry = false) => {
-    const rootUrl = 'http://web.gzh360.com';
-    const currentUrl = `${rootUrl}/${id ? `${path}?id=${id}` : ''}`;
-
-    const respData = cacheInitEntry ? await ctx.cache.tryGet(currentUrl, async () => await getInitEntry(currentUrl), config.cache.routeExpire, false) : await getInitEntry(currentUrl);
-    const $ = cheerio.load(respData);
-
-    const title = id ? $('head > title').text().split(' - ', 1)[0] : '首页';
-
-    let items = $('div.content div.news_desc')
-        .map((_, item) => {
-            item = $(item);
-            const link = item.find('h3 > a');
-            const href = link.attr('href');
-            const title = link.text();
-            const pubDate = item.find('span.datecss > span').attr('data-timestamp');
-            let author;
-            if (!skipAuthor) {
-                author = item.find('span.fromcss > a').text(); // only some pages have this
-            }
-            return {
-                link: `${href.startsWith('http') ? '' : rootUrl}${href}`,
-                title,
-                pubDate: parseDate(pubDate),
-                author,
-            };
-        })
-        .get();
-
-    items = await Promise.all(items.map((item) => finishArticleItem(ctx, item, skipAuthor)));
-
-    ctx.state.data = {
-        title: `${titleHeader}${custom_title ?? title}`,
-        link: currentUrl,
-        item: items.filter((item) => item),
-    };
-};

package/lib/v2/gzh360/utils.js

@@ -1,62 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const { fixArticleContent } = require('@/utils/wechat-mp');
-
-const invalidIdError = new RangeError('Invalid id');
-
-const mpIdEncode = (name) => {
-    const onePassed = Buffer.from(name).toString('base64'); // one-pass uses the standard base64 alphabet
-    // two-passed base64 is always url-safe (`7f+/` never appear), you can prove it by yourself
-    return Buffer.from(onePassed).toString('base64');
-};
-
-const mpIdDecode = (id) => {
-    // verify that the decoded name can be a valid WeChat mp name: https://kf.qq.com/faq/120911VrYVrA141110r2MRJV.html
-    // "公众号名称/昵称可设置4-30个字符（1个汉字算2字符）": 4 ASCII characters => 12, 15 Chinese characters => 80
-    // and the id is a valid two-passed base64 string (`7f+/` never appear)
-    if (id.length < 12 || id.length > 80 || id.length % 4 !== 0 || !/^[a-eg-zA-Z0-68-9]+={0,2}$/.test(id)) {
-        throw invalidIdError;
-    }
-    const deSecondPassed = Buffer.from(id, 'base64').toString();
-    // verify that it is a valid base64 string using standard base64 alphabet
-    if (deSecondPassed.length % 4 !== 0 || !/^[a-zA-Z0-9+/]+={0,2}$/.test(deSecondPassed)) {
-        throw invalidIdError;
-    }
-    const deFirstPassed = Buffer.from(deSecondPassed, 'base64').toString();
-    // "空格不可在最前或者最后，且空格不可连续"
-    if (deFirstPassed.length < 2 || deFirstPassed.startsWith(' ') || deFirstPassed.endsWith(' ') || deFirstPassed.includes('  ')) {
-        throw invalidIdError;
-    }
-    return deFirstPassed;
-};
-
-const finishArticleItem = async (ctx, item, skipAuthor = false) => {
-    // the website is slow and unstable, so we need to ignore errors to avoid breaking the whole route
-    // we can't cache the item instead of the webpage because we need to retry when the last request failed
-    const article = await ctx.cache.tryGet(
-        item.link,
-        async () =>
-            await got(item.link)
-                .then((_r) => _r.data)
-                .catch(() => null) // it is safe do that in tryGet because a false value always lead to a cache miss
-    );
-
-    if (article) {
-        const $ = cheerio.load(article);
-        if (!skipAuthor) {
-            item.author = item.author || $('[id=webbdgzh]+a').text();
-        }
-        item.title = item.title || $('div.desc > h1').text();
-        item.pubDate = item.pubDate || parseDate($('div.desc span[data-timestamp]').attr('data-timestamp'));
-        item.description = fixArticleContent($('div.rich_media_content'), true); // sometimes it is an empty string due to the website's fault
-    }
-
-    return item;
-};
-
-module.exports = {
-    mpIdEncode,
-    mpIdDecode,
-    finishArticleItem,
-};

package/lib/v2/hellobtc/topic.js

@@ -1,43 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
-
-const rootUrl = 'https://www.hellobtc.com';
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id;
-    const url = `${rootUrl}/topic/${id}.html`;
-
-    const response = await got(url);
-    const $ = cheerio.load(response.data);
-    const title = $('div.content > h1').text();
-    const list = $('div.tab_pane')
-        .find('div.new_item')
-        .map((_, item) => ({
-            title: $(item).find('h2').text(),
-            link: $(item).find('a').attr('href'),
-        }))
-        .filter((item) => item)
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got(item.link);
-                const content = cheerio.load(detailResponse.data);
-
-                item.description = content('#nr').html();
-                item.pubDate = timezone(parseDate(content('span.date').text(), 'YYYY-MM-DD HH:mm:ss'), +8);
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: `白话区块链 - ${title}专栏`,
-        link: url,
-        item: items,
-    };
-};

package/lib/v2/hellogithub/article.js

@@ -1,47 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
-
-module.exports = async (ctx) => {
-    const rootUrl = 'https://hellogithub.com';
-    const currentUrl = 'https://hellogithub.com/article';
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-    const $ = cheerio.load(response.data);
-    const list = $('.content .post')
-        .slice(0, 10)
-        .map((_, item) => {
-            const a = $(item).find('a.post-title').eq(0).eq(0);
-            const link = rootUrl + a.attr('href');
-            return {
-                title: a.text(),
-                link,
-                description: $(item).find('.post-description')[0].children[0].data,
-                pubDate: timezone(parseDate($(item).find('.post-meta').text(), 'YYYY-MM-DD HH:mm:ss'), +8),
-            };
-        })
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-                const content = cheerio.load(detailResponse.data);
-                item.description = content('.markdown-body').html();
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: 'HelloGitHub - Article',
-        link: currentUrl,
-        item: items,
-    };
-};

package/lib/v2/hellogithub/month.js

@@ -1,42 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const rootUrl = 'https://hellogithub.com';
-    const currentUrl = 'https://hellogithub.com/article';
-    const response = await got({
-        method: 'get',
-        url: rootUrl,
-    });
-    const $ = cheerio.load(response.data);
-    let count = $('.pricing-table-price').eq(0).text();
-    count = count.replace(/[^0-9]/gi, '');
-    let start = count - 10;
-    const PromiseArr = [];
-    for (start; start <= count; start++) {
-        PromiseArr.push(
-            new Promise((resolve) => {
-                const link = `https://hellogithub.com/periodical/volume/${start}/`;
-                const issueNum = start;
-                return ctx.cache.tryGet(link, async () => {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: link,
-                    });
-                    const $ = cheerio.load(detailResponse.data);
-                    return resolve({
-                        title: '第' + issueNum + '期',
-                        link,
-                        description: $('.content').html(),
-                    });
-                });
-            })
-        );
-    }
-    const items = await Promise.all(PromiseArr);
-    ctx.state.data = {
-        title: 'HelloGitHub - Article',
-        link: currentUrl,
-        item: items,
-    };
-};

package/lib/v2/hellogithub/ranking.js

@@ -1,39 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    let type;
-    switch (ctx.params.type) {
-        case 'db':
-            type = 'db-engines';
-            break;
-        case 'webserver':
-            type = 'netcraft';
-            break;
-        default:
-            type = 'tiobe';
-            break;
-    }
-    const rootUrl = 'https://hellogithub.com/report/' + type;
-    ctx.state.data = {
-        title: 'HelloGitHub - ranking',
-        link: rootUrl,
-        item: await ctx.cache.tryGet(rootUrl, async () => {
-            const response = await got({
-                method: 'get',
-                url: rootUrl,
-            });
-            const $ = cheerio.load(response.data);
-            const content = $('.content');
-            const title = $('.header').find('h1');
-
-            return [
-                {
-                    title: title.text(),
-                    link: rootUrl,
-                    description: content.html(),
-                },
-            ];
-        }),
-    };
-};

package/lib/v2/heu/gx/card.js

@@ -1,47 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-const rootUrl = 'http://news.hrbeu.edu.cn';
-
-module.exports = async (ctx) => {
-    const date = new Date();
-    const hour = date.getHours();
-    const minute = date.getMinutes();
-    const column = ctx.params.column;
-    const id = ctx.params.id || '';
-    let toUrl;
-    if (id !== '') {
-        toUrl = rootUrl.concat('/', column, '/', id, '.htm');
-    } else {
-        toUrl = rootUrl.concat('/', column, '.htm');
-    }
-
-    const response = await got({
-        method: 'get',
-        url: toUrl,
-        headers: {
-            Referer: rootUrl,
-        },
-    });
-
-    const $ = cheerio.load(response.data);
-
-    const bigTitle = $('div.list-left-tt')
-        .text()
-        .replace(/[ ]|[\r\n]/g, '');
-
-    const card = $('li.clearfix')
-        .map((_, item) => ({
-            title: $(item).find('div.list-right-tt').text(),
-            pubDate: $(item).find('span').text() + ' ' + hour + ':' + minute,
-            link: $(item).find('a').attr('href'),
-            description: $(item).find('div.list-right-p').text(),
-        }))
-        .get();
-
-    ctx.state.data = {
-        title: '工学-' + bigTitle,
-        link: toUrl,
-        item: card,
-    };
-};