rsshub 1.0.0-master.ff1cc11 → 1.0.0-master.ff97759

package/lib/routes/github/starred_repos.js

@@ -1,63 +0,0 @@
-const got = require('@/utils/got');
-const config = require('@/config').value;
-
-module.exports = async (ctx) => {
-    if (!config.github || !config.github.access_token) {
-        throw 'GitHub star RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#%E9%83%A8%E5%88%86-rss-%E6%A8%A1%E5%9D%97%E9%85%8D%E7%BD%AE">relevant config</a>';
-    }
-    const user = ctx.params.user;
-
-    const host = `https://github.com/${user}?tab=stars`;
-    const url = 'https://api.github.com/graphql';
-
-    const response = await got({
-        method: 'post',
-        url,
-        headers: {
-            Authorization: `bearer ${config.github.access_token}`,
-        },
-        json: {
-            query: `
-            {
-                user(login: "${user}") {
-                  starredRepositories(first: 10, orderBy: {direction: DESC, field: STARRED_AT}) {
-                    edges {
-                      starredAt
-                      node {
-                        name
-                        description
-                        url
-                        openGraphImageUrl
-                        primaryLanguage {
-                          name
-                        }
-                        stargazers {
-                          totalCount
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            `,
-        },
-    });
-
-    const data = response.data.data.user.starredRepositories.edges;
-
-    ctx.state.data = {
-        allowEmpty: true,
-        title: `${user}’s starred repositories`,
-        link: host,
-        description: `${user}’s starred repositories`,
-        item: data.map((repo) => ({
-            title: `${user} starred ${repo.node.name}`,
-            author: user,
-            description: `${repo.node.description === null ? 'no description' : repo.node.description} <br> primary language: ${
-                repo.node.primaryLanguage === null ? 'no primary language' : repo.node.primaryLanguage.name
-            } <br> stargazers: ${repo.node.stargazers.totalCount} <br> <img sytle="width:50px;" src='${repo.node.openGraphImageUrl}'>`,
-            pubDate: new Date(repo.starredAt),
-            link: repo.node.url,
-        })),
-    };
-};

package/lib/routes/github/trending.js

@@ -1,43 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const since = ctx.params.since;
-    const language = ctx.params.language || '';
-    const spoken_language = ctx.params.spoken_language || '';
-    const url = `https://github.com/trending/${encodeURIComponent(language)}?since=${since}&spoken_language_code=${spoken_language}`;
-
-    const response = await got({
-        method: 'get',
-        url,
-        headers: {
-            Referer: url,
-        },
-    });
-
-    const data = response.data;
-
-    const $ = cheerio.load(data);
-    const list = $('article');
-
-    ctx.state.data = {
-        title: $('title').text(),
-        link: url,
-        item:
-            list &&
-            list
-                .map((_, item) => {
-                    item = $(item);
-                    return {
-                        title: item.find('h1').text(),
-                        author: item.find('h1').text().split('/')[0].trim(),
-                        description: `${item.find('.pr-4').text()}<br>
-                            <br>Language: ${item.find('span[itemprop="programmingLanguage"]').text() || 'unknown'}
-                            <br>Star: ${item.find('.Link--muted').eq(0).text().trim()}
-                            <br>Fork: ${item.find('.Link--muted').eq(1).text().trim()}`,
-                        link: `https://github.com${item.find('h1 a').attr('href')}`,
-                    };
-                })
-                .get(),
-    };
-};

package/lib/routes/gov/fmprc/fyrbt.js

@@ -1,24 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const util = require('./utils');
-
-const host = 'https://www.fmprc.gov.cn/web/wjdt_674879/fyrbt_674889/';
-
-module.exports = async (ctx) => {
-    const response = await got.get(host, {
-        responseType: 'buffer',
-    });
-
-    const $ = cheerio.load(response.data);
-
-    const list = $('.rebox_news ul li').get();
-
-    const result = await util.ProcessFeed(list, ctx.cache);
-
-    ctx.state.data = {
-        title: '中华人民共和国外交部-发言人表态',
-        link: host,
-        description: '中华人民共和国外交部-发言人表态',
-        item: result,
-    };
-};

package/lib/routes/gov/fmprc/utils.js

@@ -1,54 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = require('url');
-
-// 加载文章页
-async function load(link) {
-    const response = await got.get(link);
-    const $ = cheerio.load(response.data);
-
-    const date = new Date(
-        $('#News_Body_Time')
-            .text()
-            .match(/\d{4}-\d{2}-\d{2}/)
-    );
-
-    const pubDate = new Date(date.getTime()).toUTCString();
-
-    // 提取内容
-    const description = $('#News_Body_Txt_A').html();
-
-    return { description, pubDate };
-}
-
-const ProcessFeed = (list, caches) => {
-    const host = 'https://www.fmprc.gov.cn/web/wjdt_674879/fyrbt_674889/';
-
-    return Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-
-            const $title = $('a');
-            // 还原相对链接为绝对链接
-            const itemUrl = url.resolve(host, $title.attr('href'));
-
-            // 列表上提取到的信息
-            const single = {
-                title: $.text(),
-                link: itemUrl,
-                guid: itemUrl,
-            };
-
-            // 使用tryGet方法从缓存获取内容。
-            // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
-            const other = await caches.tryGet(itemUrl, () => load(itemUrl));
-
-            // 合并解析后的结果集作为该篇文章最终的输出结果
-            return Promise.resolve(Object.assign({}, single, other));
-        })
-    );
-};
-
-module.exports = {
-    ProcessFeed,
-};

package/lib/routes/gov/mee/gs.js

@@ -1,54 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const linkBase = `http://www.mee.gov.cn/xxgk/gs/gsq/`;
-    const listData = await got.get(linkBase);
-    const $ = cheerio.load(listData.data);
-    ctx.state.data = {
-        title: `公示 - 中华人民共和国生态环境部`,
-        link: linkBase,
-        item: await Promise.all(
-            $('.main .main_top li')
-                .map(async (_, el) => {
-                    const $el = $(el);
-                    const $a = $el.find('>a');
-                    const href = $a.attr('href');
-                    const key = `gov_gs: ${href}`;
-                    let description;
-                    const value = await ctx.cache.get(key);
-
-                    // 移除 href 中的 ./，并且拼接原来的 url
-                    const link = `${linkBase}${href.slice(2)}`;
-
-                    if (value) {
-                        description = value;
-                    } else {
-                        const contentData = await got.get(link);
-                        const $content = cheerio.load(contentData.data);
-                        description = $content('.TRS_Editor').html();
-                        ctx.cache.set(key, description);
-                    }
-
-                    const title = $a.text();
-
-                    // 获取 date
-                    const pubDate = new Date(
-                        $el
-                            .find('.shover')
-                            .first()
-                            .text()
-                            .match(/\d{4}-\d{2}-\d{2}/)
-                    ).toUTCString();
-
-                    return {
-                        title,
-                        description,
-                        link,
-                        pubDate,
-                    };
-                })
-                .get()
-        ),
-    };
-};

package/lib/routes/gov/moe/moe.js

@@ -1,84 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const logger = require('@/utils/logger');
-const liburl = require('url');
-
-const moeUrl = 'http://www.moe.gov.cn/';
-const typesIdMap = [
-    { type: 'policy_anal', id: 'tt_con2', name: '政策解读' },
-    { type: 'newest_file', id: 'nine_con1', name: '最新文件' },
-    { type: 'notice', id: 'nine_con2', name: '公告公示' },
-    { type: 'edu_ministry_news', id: 'nine_con3', name: '教育部简报' },
-];
-
-module.exports = async (ctx) => {
-    const type = ctx.params.type;
-    let id = '';
-    let name = '';
-
-    for (let i = 0; i < typesIdMap.length; i++) {
-        if (typesIdMap[i].type === type) {
-            id = typesIdMap[i].id;
-            name = typesIdMap[i].name;
-        }
-    }
-
-    if (id === '') {
-        logger.error('The given type not found.');
-        return;
-    }
-
-    const response = await got({
-        method: 'get',
-        url: moeUrl,
-        Host: 'www.moe.gov.cn',
-    });
-
-    const $ = cheerio.load(response.data);
-    const newsLis = $('div#' + id + '>ul>li');
-
-    ctx.state.data = {
-        title: name,
-        link: moeUrl,
-        item: await Promise.all(
-            newsLis
-                .map(async (_, item) => {
-                    item = $(item);
-
-                    const firstA = item.find('a');
-                    const itemUrl = liburl.resolve(moeUrl, firstA.attr('href'));
-
-                    const infos = await ctx.cache.tryGet(itemUrl, async () => {
-                        const res = {};
-                        const response = await got({
-                            method: 'get',
-                            url: itemUrl,
-                            Host: 'www.moe.gov.cn',
-                        });
-                        const data = cheerio.load(response.data);
-
-                        if (itemUrl.indexOf('www.gov.cn') > -1) {
-                            res.description = data('div.pages_content > p').html();
-                            res.title = data('div.article > h1').text();
-                        } else if (itemUrl.indexOf('srcsite') > -1) {
-                            res.description = data('div#xxgk_content_div').html();
-                            res.title = data('div#xxgk_content_redheadbg > h1').text();
-                        } else if (itemUrl.indexOf('jyb_') > -1) {
-                            res.description = data('div.TRS_Editor').html();
-                            res.title = data('div#content_body > h1').text();
-                        }
-
-                        return res;
-                    });
-
-                    return {
-                        title: infos.title,
-                        description: infos.description,
-                        link: itemUrl,
-                        pubDate: `${new Date().getFullYear()}-${item.find('span').text()}`,
-                    };
-                })
-                .get()
-        ),
-    };
-};

package/lib/routes/guancha/headline.js

@@ -1,48 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const rootUrl = 'https://www.guancha.cn';
-    const currentUrl = `${rootUrl}/GuanChaZheTouTiao/list_1.shtml`;
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-
-    const $ = cheerio.load(response.data);
-
-    const list = $('.headline-list li .content-headline h3 a')
-        .map((_, item) => {
-            item = $(item);
-            return {
-                title: item.text(),
-                link: `${rootUrl}${item.attr('href')}`,
-                description: item.parent().next().html(),
-                pubDate: new Date(item.parents('div').eq(0).find('span').text()).toUTCString(),
-            };
-        })
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-
-                const content = cheerio.load(detailResponse.data);
-
-                item.description += content('.all-txt').html();
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: '观察者网 - 头条',
-        link: currentUrl,
-        item: items,
-    };
-};

package/lib/routes/guancha/index.js

@@ -1,132 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const date = require('@/utils/date');
-
-const config = {
-    review: { title: '评论 & 研究', query: '.module-news-main' },
-    story: { title: '要闻', query: '.img-List' },
-    fengwen: { title: '风闻', query: '.fengwen-list' },
-    redian: { title: '热点' },
-    gundong: { title: '滚动' },
-    all: { title: '全部' },
-    home: { title: '首页' },
-    others: { title: '热点 & 滚动' },
-};
-
-module.exports = async (ctx) => {
-    const total = 10;
-    const caty = ctx.params.caty || 'all';
-    const rootUrl = 'https://www.guancha.cn';
-
-    let newsList = [],
-        redianList = [],
-        gundongList = [];
-
-    // 'review', 'story' and 'fengwen' come from homepage.
-
-    if (caty === 'review' || caty === 'story' || caty === 'fengwen' || caty === 'all' || caty === 'home') {
-        const response = await got({
-            method: 'get',
-            url: rootUrl,
-        });
-
-        const $ = cheerio.load(response.data);
-
-        const fetchPost = (slice) =>
-            slice
-                .find('h4.module-title a')
-
-                // Filter some blank links which lead to no contents but 'https://user.guancha.cn'.
-
-                .filter((_, item) => $(item).attr('href') !== 'https://user.guancha.cn')
-                .map((_, item) => {
-                    item = $(item);
-                    const link = item.attr('href');
-                    return {
-                        title: item.text(),
-                        link: (link.indexOf('http') === 0 ? '' : rootUrl) + link,
-                    };
-                })
-                .get();
-
-        if (caty === 'all' || caty === 'home') {
-            newsList = fetchPost($(config.review.query))
-                .slice(0, total / 3)
-                .concat(fetchPost($(config.story.query)).slice(0, total / 3), fetchPost($(config.fengwen.query)).slice(0, total / 3));
-        } else {
-            newsList = fetchPost($(config[caty].query)).slice(0, total);
-        }
-    }
-
-    // 'redian' and 'gundong' come from api.
-
-    if (caty === 'redian' || caty === 'all' || caty === 'others') {
-        const response = await got({
-            method: 'get',
-            url: `${rootUrl}/api/redian.htm`,
-        });
-        redianList = response.data.items
-            .map((item) => ({
-                title: item.TITLE,
-                link: `${rootUrl}${item.HTTP_URL}`,
-            }))
-            .slice(0, caty === 'all' ? total / 3 : total);
-    }
-    if (caty === 'gundong' || caty === 'all' || caty === 'others') {
-        const response = await got({
-            method: 'get',
-            url: `${rootUrl}/api/gundong.htm`,
-        });
-        gundongList = response.data.items
-            .map((item) => ({
-                title: item.TITLE,
-                link: `${rootUrl}${item.HTTP_URL}`,
-            }))
-            .slice(0, caty === 'all' ? total / 3 : total);
-    }
-
-    const items = await Promise.all(
-        newsList.concat(redianList, gundongList).map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                let detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-
-                // Some links to posts in 'fengwen' must be redirected in order to fetch content.
-                // eg. https://www.guancha.cn/politics/2020_10_23_569021.shtml
-                //  => https://user.guancha.cn/main/content?id=399176
-
-                const jumpMatch = detailResponse.data.match(/user.guancha.cn\/main\/content\?id=(.*)";/);
-                if (jumpMatch !== null) {
-                    item.link = `https://user.guancha.cn/main/content?id=${jumpMatch[1]}`;
-                    detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                    });
-                }
-
-                const content = cheerio.load(detailResponse.data);
-
-                const dateMatch = detailResponse.data.match(/"pubDate": "(.*)"/);
-                if (dateMatch === null) {
-                    // PubDates of posts in 'fengwen' are in an informal format.
-
-                    item.pubDate = date(content('.time1').text(), 8);
-                } else {
-                    item.pubDate = new Date(dateMatch[1]).toUTCString();
-                }
-
-                item.description = content('.all-txt').html() || content('.article-txt-content').html();
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: `观察者网 - ${config[caty].title}`,
-        link: rootUrl,
-        item: items,
-    };
-};

package/lib/routes/guancha/member.js

@@ -1,71 +0,0 @@
-const got = require('@/utils/got');
-
-const title = {
-    recommend: '精选',
-    books: '观书堂',
-    courses: '在线课',
-    huodongs: '观学院',
-};
-
-module.exports = async (ctx) => {
-    const caty = ctx.params.caty || 'recommend';
-
-    const rootUrl = 'https://member.guancha.cn';
-    const apiUrl = `${rootUrl}/zaixianke/home`;
-    const response = await got({
-        method: 'get',
-        url: apiUrl,
-    });
-
-    let items;
-
-    switch (caty) {
-        case 'books':
-            items = response.data.data.books.map((item) => ({
-                title: item.title,
-                link: `${rootUrl}/guanshutang/summary.html?id=${item.id}`,
-                description: `<img src="${item.cover}"><p>[${item.audio_time}] ${item.desc_short}</p>`,
-                pubDate: new Date(parseInt(item.cover.split('/').pop().substr(0, 10)) * 1000).toUTCString(),
-            }));
-            break;
-
-        case 'courses':
-            items = response.data.data.courses.data.map((item) => {
-                let description = '',
-                    pubDate = new Date(0);
-
-                for (const i of item.items) {
-                    const newPubDate = new Date(i.publish_time);
-                    pubDate = pubDate > newPubDate ? pubDate : newPubDate;
-                    description += `<a href="${rootUrl}/zaixianke/content.html?id=${i.id}">${i.title}</a><br>`;
-                }
-
-                return {
-                    title: item.name,
-                    link: `${rootUrl}/zaixianke/summary.html?id=${item.id}`,
-                    author: item.author_name,
-                    description: `<img src="${item.cover}"><p>${item.desc_short}</p><br>${description}`,
-                    pubDate: pubDate.toUTCString(),
-                };
-            });
-            break;
-
-        default:
-            items = response.data.data[caty].map((item) => ({
-                title: item.title,
-                link: item.jump_url,
-                author: item.author_name,
-                description: `<img src="${item.big_pic}"><p>${item.summary}</p>`,
-                enclosure_url: item.media_url,
-                enclosure_length: item.media_size,
-                enclosure_type: 'audio/mpeg',
-                pubDate: new Date(item.created_at * 1000).toUTCString(),
-            }));
-    }
-
-    ctx.state.data = {
-        title: `观学院 - ${title[caty]}`,
-        link: `${rootUrl}/index.html`,
-        item: items,
-    };
-};

package/lib/routes/guancha/topic.js

@@ -1,55 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const date = require('@/utils/date');
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id || '0';
-    const order = ctx.params.order || '1';
-
-    const rootUrl = 'https://user.guancha.cn';
-    const currentUrl = `${rootUrl}/${id === '0' ? 'main/index-list.json?' : 'topic/post-list?topic_id=' + id}&page=1&order=${order}`;
-
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-
-    const $ = cheerio.load(response.data);
-
-    const list = $('.list-item h4 a, ul.home li .list-item h4 a')
-        .slice(0, 10)
-        .map((_, item) => {
-            item = $(item);
-            return {
-                title: item.text(),
-                link: `${rootUrl}${item.attr('href')}`,
-            };
-        })
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-                const content = cheerio.load(detailResponse.data);
-
-                item.pubDate = date(content('.time1').text(), 8);
-                item.author = content('.user-main h4 a').eq(0).text();
-                item.description = content('.article-txt-content').html();
-
-                return item;
-            })
-        )
-    );
-
-    $('h1.title span').remove();
-
-    ctx.state.data = {
-        title: `观察者网 - ${id === '0' ? '风闻' : $('h1.title').text()}`,
-        link: currentUrl,
-        item: items,
-    };
-};

package/lib/routes/guokr/calendar.js

@@ -1,47 +0,0 @@
-const got = require('@/utils/got');
-
-async function loadFullPage(ctx, id) {
-    const link = `https://apis.guokr.com/minisite/article/${id}.json`;
-    const content = await ctx.cache.tryGet(link, async () => {
-        const res = await got.get(link);
-        return res.data.result.content;
-    });
-    return content;
-}
-
-const channelMap = {
-    calendar: 'pac',
-    institute: 'predator',
-    foodlab: 'predator',
-    pretty: 'beauty',
-};
-
-module.exports = async (ctx) => {
-    const channel = channelMap[ctx.params.channel] ? channelMap[ctx.params.channel] : ctx.params.channel;
-
-    const response = await got.get(`https://www.guokr.com/apis/minisite/article.json?retrieve_type=by_wx&channel_key=${channel}&offset=0&limit=10`);
-    const items = response.data.result;
-
-    if (items.length === 0) {
-        throw 'Unknown channel';
-    }
-
-    const channel_name = items[0].channels[0].name;
-    const channel_url = items[0].channels[0].url;
-
-    const result = await Promise.all(
-        items.map(async (item) => ({
-            title: item.title,
-            description: await loadFullPage(ctx, item.id), // Mercury 无法正确解析全文，故这里手动加载
-            pubDate: item.date_published,
-            link: item.url,
-            author: item.author.nickname,
-        }))
-    );
-
-    ctx.state.data = {
-        title: `果壳网 ${channel_name}`,
-        link: channel_url,
-        item: result,
-    };
-};