rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/routes/initium/full.js

@@ -1,188 +0,0 @@
-const got = require('@/utils/got');
-const config = require('@/config').value;
-const cheerio = require('cheerio');
-const resolve_url = require('url').resolve;
-const timezone = require('@/utils/timezone');
-
-module.exports = async (ctx) => {
-    // model是channel/tag/etc.，而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话，那就是model是follow，type是articles。
-    const model = ctx.params.model || 'channel';
-    const type = ctx.params.type || 'latest';
-    const language = ctx.params.language || 'zh-hans';
-    let listurl;
-    let listlink;
-    switch (model) {
-        case 'author':
-            listurl = `https://api.theinitium.com/api/v2/author/?language=${language}&slug=${type}`;
-            listlink = `https://theinitium.com/author/${type}/`;
-            break;
-        case 'follow':
-            listurl = `https://api.theinitium.com/api/v2/user/follows/${type}/?language=${language}`;
-            listlink = `https://theinitium.com/follow/`;
-            break;
-        case 'channel':
-            listurl = `https://api.theinitium.com/api/v2/channel/articles/?language=${language}&slug=${type}`;
-            listlink = `https://theinitium.com/channel/${type}/`;
-            break;
-        case 'tags':
-            listurl = `https://api.theinitium.com/api/v2/tag/articles/?language=${language}&slug=${type}`;
-            listlink = `https://theinitium.com/tags/${type}/`;
-            break;
-    }
-
-    const key = {
-        email: config.initium.username,
-        password: config.initium.password,
-        iapReceipt: config.initium.iap_receipt,
-    };
-    const body = JSON.stringify(key);
-
-    let token;
-    const cache = await ctx.cache.get('INITIUM_TOKEN');
-    if (cache) {
-        token = cache;
-    } else if (config.initium.bearertoken) {
-        token = config.initium.bearertoken;
-        ctx.cache.set('INITIUM_TOKEN', config.initium.bearertoken);
-    } else if (key.email === undefined) {
-        token = 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY=';
-    } else {
-        const login = await got({
-            method: 'post',
-            url: `https://api.theinitium.com/api/v2/auth/login/?language=${language}`,
-            headers: {
-                'Content-Type': 'application/json',
-                Accept: 'application/json',
-                Connection: 'keep-alive',
-                Authorization: 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY=',
-                Origin: `https://theinitium.com/`,
-                Referer: `https://theinitium.com/`,
-                'X-Client-Name': 'Web',
-            },
-            body,
-        });
-
-        /*
-        const devices = login.data.access.devices;
-
-        for (const key in devices) {
-            const device = devices[key];
-            if (device.status === 'logged_in' && !device.logout_at && device.platform === 'web') {
-                token = 'Bearer ' + device.device_id;
-                break;
-            }
-        }
-        */
-        token = 'Bearer ' + login.data.token;
-        ctx.cache.set('INITIUM_TOKEN', token);
-    }
-
-    const headers = {
-        Accept: '*/*',
-        Connection: 'keep-alive',
-        Authorization: token,
-        'X-IAP-Receipt': key.iapReceipt || '',
-    };
-
-    const response = await got({
-        method: 'get',
-        url: listurl,
-        headers,
-    });
-
-    const name = response.data.name || (response.data[model] && response.data[model].name) || '追踪';
-    // 从v1直升的channel和tags里面是digests，v2新增的author和follow出来都是results
-    const articles = response.data.results ? response.data.results : response.data.digests;
-    // 如果model=author，那就是avatar；否则都是cover，要么就没封面
-    const image = response.data[model] && (response.data[model].cover || response.data[model].avatar);
-
-    const getFullText = async (slug) => {
-        let content = await ctx.cache.get(slug + language);
-        if (content) {
-            return content;
-        }
-
-        content = '';
-
-        const response = await got({
-            method: 'get',
-            url: `https://api.theinitium.com/api/v2/article/detail/?language=${language}&slug=${slug}`,
-            headers,
-        });
-
-        const data = response.data;
-
-        if (data.lead.length) {
-            content += '<p>「' + data.lead + '」</p>';
-        }
-        if (data.byline.length) {
-            content += '<p>' + data.byline + '</p>';
-        }
-        if (data.content) {
-            content += data.content.replace('<figure class="advertisement"/><br/>', '').replace(/<br><br>-----------<br>.*$/g, '');
-        } else if (data.type === 'html') {
-            // 有时候编辑部会漏录入文章信息…………扶额。所以加这一个判断，如果确实是普通html文章，但又没有内容，说明是漏了，后面还要给guid手动加个标记，以便阅读器事后重抓。
-            content += '内容为空，请稍后再来';
-        } else if (data.type === 'web') {
-            // 有时候文章并非普通html文章，而是带有互动内容等，表现为type为web，并且content里没有内容。我们也尽力抓点东西下来。
-            // 或许可能还有其他未知情况，等碰到了再说吧。先这样留空也不碍事。
-            const nonhtmlcontent = await got({
-                method: 'get',
-                url: data.web.url,
-            });
-            const webcontent = cheerio.load(nonhtmlcontent.body).html();
-            content += webcontent;
-        }
-        if (data.paywall_enabled) {
-            const google_bot_ua =
-                'Mozilla / 5.0(Linux; Android 6.0.1; Nexus 5X Build / MMB29P) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.92 Mobile Safari / 537.36(compatible; Googlebot / 2.1; +http://www.google.com/bot.html)';
-            const accept_language = language + ';q=0.9';
-            const pay_part = await got({
-                method: 'get',
-                url: `https://theinitium.com/article/${slug}`,
-                headers: {
-                    'user-agent': google_bot_ua,
-                    'accept-language': accept_language,
-                },
-            });
-            const $ = cheerio.load(pay_part.body);
-            const pay_content = $('div.paywall').html();
-            if (pay_content) {
-                content += pay_content.replace('<meta itemprop="isAccessibleForFree" content="false">', '');
-            }
-        }
-
-        ctx.cache.set(slug + language, content);
-
-        return content;
-    };
-
-    const items = await Promise.all(
-        articles
-            .filter((a) => a.article)
-            .slice(0, token === 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY=' && key.iapReceipt === undefined ? 10 : articles.length)
-            .map(async (item) => {
-                item.article.date = timezone(item.article.date, +8);
-                item.article.updated = timezone(item.article.updated, +8);
-                const description = await getFullText(item.article.slug);
-                return {
-                    title: item.article.headline,
-                    author: item.article.authors.length > 0 ? item.article.authors.map((x) => x.name).toString() : item.article.byline,
-                    category: item.article.channels.filter((x) => !x.homepage).map((x) => x.name),
-                    description,
-                    link: resolve_url('https://theinitium.com', item.article.url),
-                    pubDate: item.article.date,
-                    updated: item.article.updated,
-                    // 如果遇到编辑部漏录入情况，则给uuid做个手脚，以便阅读器到时重抓。
-                    guid: description.endsWith('内容为空，请稍后再来') ? item.article.uuid + '-I-am-empty' : item.article.uuid,
-                };
-            })
-    );
-
-    ctx.state.data = {
-        title: `端传媒 - ${name}`,
-        link: listlink,
-        item: items,
-        image,
-    };
-};

package/lib/routes/kuaibao/index.js

@@ -1,33 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const currentUrl = 'http://kuaibao.qq.com/n/getKbMainpageInfo';
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-
-    const articleList = response.data.newslist.map((item) => ({
-        title: item.title,
-        link: `https://kuaibao.qq.com/getSubNewsContent?id=${item.id}&style=json`,
-        pubDate: new Date(item.time).toUTCString(),
-    }));
-
-    ctx.state.data = {
-        title: '看点快报 - 首页',
-        link: 'http://kuaibao.qq.com/',
-        item: await Promise.all(
-            articleList.map((item) =>
-                ctx.cache.tryGet(item.link, async () => {
-                    const contentResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                    });
-                    item.link = contentResponse.data.url || contentResponse.data.short_url;
-                    item.description = contentResponse.data.content.text;
-                    return item;
-                })
-            )
-        ),
-    };
-};

package/lib/routes/langlive/room.js

@@ -1,33 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id;
-    const url = `https://play.lang.live/${id}`;
-
-    const api = `https://game-api.lang.live/webapi/v1/room/info?room_id=${id}`;
-    const response = await got({
-        method: 'get',
-        url: api,
-    });
-
-    let item = [];
-    let name = '';
-    if (response.data.data.live_info.live_status === 1) {
-        name = response.data.data.live_info.nickname;
-        item = [
-            {
-                title: response.data.data.live_info.nickname + '开播了',
-                link: url,
-                guid: response.data.data.live_info.live_id,
-                description: response.data.data.live_info.describe,
-            },
-        ];
-    }
-
-    ctx.state.data = {
-        title: `${name}的浪Play直播`,
-        link: url,
-        item,
-        allowEmpty: true,
-    };
-};

package/lib/routes/logseq/changelog.js

@@ -1,42 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const rootUrl = 'https://logseq.com';
-    const currentUrl = `${rootUrl}/blog/changelog`;
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-
-    const data = response.data.split('<p class="footer">')[0];
-
-    const list = [],
-        versionsIndices = [];
-    const versions = data.match(
-        /<div class="block"><div id=""><span class="page-reference"><a class="page-ref" href="[a-z]{3}%20\d+[a-z]{2}%2C%20\d{4}">[a-zA-Z]{3} \d+[a-z]{2}, \d{4}<\/a><\/span><\/div><p><b>Version \d\.\d\.\d\.\d/g
-    );
-
-    for (const version of versions) {
-        versionsIndices.push(data.indexOf(version));
-    }
-
-    for (let i = 0; i < versionsIndices.length; i++) {
-        list.push(data.slice(versionsIndices[i], versionsIndices[i + 1]));
-    }
-
-    const items = list.map((item, index) => {
-        const version = versions[index];
-        return {
-            link: currentUrl,
-            description: item,
-            title: version.match(/Version \d\.\d\.\d\.\d/)[0],
-            pubDate: new Date(version.match(/[a-zA-Z]{3} \d+[a-z]{2}, \d{4}/)[0].replace(/st|nd|rd|th/, '')).toUTCString(),
-        };
-    });
-
-    ctx.state.data = {
-        title: 'Changelog - Logseq',
-        link: currentUrl,
-        item: items,
-    };
-};

package/lib/routes/luogu/contest.js

@@ -1,28 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const resolve_url = require('url').resolve;
-
-module.exports = async (ctx) => {
-    const link = 'https://www.luogu.com.cn/';
-    const response = await got.get(link);
-    const $ = cheerio.load(response.data);
-    const title = '洛谷近期比赛';
-
-    const out = $('.am-panel-hd ')
-        .slice(0, 10)
-        .map(function () {
-            const info = {
-                title: $(this).find('a').text() || $(this).text(),
-                description: $(this).html() + $(this).parent().find('.am-panel-bd').html(),
-                link: resolve_url('https://www.luogu.com.cn', $(this).find('a').attr('href')),
-            };
-            return info;
-        })
-        .get();
-
-    ctx.state.data = {
-        title,
-        link,
-        item: out,
-    };
-};

package/lib/routes/luogu/daily.js

@@ -1,53 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id || 179788;
-    const link = `https://www.luogu.org/discuss/show/${id}`;
-    const response = await got.get(link);
-    const $ = cheerio.load(response.data);
-    const title = $('title').text();
-
-    const out = $('div.am-comment-main > div > p')
-        .slice(0, 10)
-        .map(function () {
-            const info = {
-                title: $(this).find('strong').text() || $(this).text(),
-                link: $(this).find('a').attr('href'),
-            };
-            return info;
-        })
-        .get();
-
-    const items = await Promise.all(
-        out.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-                const content = cheerio.load(detailResponse.data);
-                const timeRegExp = new RegExp(/(?<=([1-9]\d{3}-(0[1-9]|1[0-2])-(0[1-9]|[1-2][0-9]|3[0-1])\s+(20|21|22|23|[0-1]\d):[0-5]\d:[0-5]\d))/);
-
-                item.description = content('#article-content').html();
-
-                content('#article-content').remove();
-
-                const line = content
-                    .html()
-                    .split('\n')
-                    .find((line) => timeRegExp.test(line));
-
-                item.pubDate = new Date(line.match(timeRegExp)[1] + ' GMT+8').toUTCString();
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title,
-        link,
-        item: items,
-    };
-};

package/lib/routes/luogu/userFeed.js

@@ -1,38 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const getUsernameFromUID = async (uid) => {
-        const key = 'luogu-username-from-uid-' + uid;
-        let name = await ctx.cache.get(key);
-        if (!name) {
-            const nameResponse = await got({
-                method: 'get',
-                url: `https://www.luogu.com.cn/user/${uid}?_contentOnly=1`,
-            });
-            name = nameResponse.data.currentData.user.name;
-            ctx.cache.set(key, name);
-        }
-        return name;
-    };
-
-    const uid = ctx.params.uid;
-    const name = await getUsernameFromUID(uid);
-    const response = await got({
-        method: 'get',
-        url: `https://www.luogu.com.cn/api/feed/list?user=${uid}`,
-    });
-    const data = response.data.feeds.result;
-
-    ctx.state.data = {
-        title: `${name} 的洛谷动态`,
-        link: `https://www.luogu.com.cn/user/${uid}#activity`,
-        allowEmpty: true,
-        item: data.map((item) => ({
-            title: item.content,
-            description: item.content,
-            pubDate: new Date(item.time * 1000).toUTCString(),
-            link: `https://www.luogu.com.cn/user/${uid}#activity`,
-            guid: item.id,
-        })),
-    };
-};

package/lib/routes/mckinsey/index.js

@@ -1,49 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const category = ctx.params.category || '';
-    const title = {
-        autos: '汽车',
-        'banking-insurance': '金融服务',
-        'business-technology': '数字化',
-        consumers: '消费者',
-        'healthcare-pharmaceuticals': '医药与医疗',
-        'mckinsey-global-institute': '麦肯锡全球研究院',
-        全球基础材料: '全球基础材料',
-        innovation: '创新',
-        macroeconomy: '宏观经济',
-        manufacturing: '制造业',
-        'talent-leadership': '人才与领导力',
-        'technology-media-and-telecom': '技术，媒体与通信',
-        'urbanization-sustainability': '城市化与可持续发展',
-        'capital-projects-infrastructure': '资本项目和基础设施',
-        交通运输与物流: '旅游、运输和物流',
-    };
-    const response = await got.get(`https://www.mckinsey.com.cn/insights/${category}`);
-    const $ = cheerio.load(response.data);
-
-    const articles = $('.fusion-column h4 a')
-        .map((index, ele) => ({
-            title: $(ele).text(),
-            link: $(ele).attr('href'),
-        }))
-        .get();
-
-    const item = await Promise.all(
-        articles.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const res = await got.get(item.link);
-                const doc = cheerio.load(res.data);
-                item.description = doc('#content').html();
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: category ? `McKinsey Greater China - ${title[category]}` : `McKinsey Greater China`,
-        link: 'https://www.mckinsey.com.cn/',
-        item,
-    };
-};

package/lib/routes/mhgui/comic.js

@@ -1,99 +0,0 @@
-const { resolve } = require('url');
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-const LZString = require('lz-string');
-
-const getChapters = ($) => {
-    let time_mark = 0;
-    // 用于一次更新多个新章节的排序
-    let new_time_mark = 0;
-    return $('h4')
-        .toArray()
-        .map((ele) => {
-            const categoryName = $(ele).text();
-            while (!$(ele.next).hasClass('chapter-list')) {
-                ele = ele.next;
-            }
-            ele = ele.next;
-            return $(ele)
-                .children('ul')
-                .toArray()
-                .reverse()
-                .reduce((acc, curr) => acc.concat($(curr).children('li').toArray()), [])
-                .map((ele) => {
-                    const a = $(ele).children('a');
-                    // 通过操作发布时间来对章节进行排序,如果是刚刚更新的单行本或者番外,保留最新更新时间
-                    let pDate = new Date(new Date($.pubDate) - time_mark++ * 1000);
-                    if (a.find('em').length > 0) {
-                        // 对更新的章节也进行排序
-                        pDate = new Date(new Date($.pubDate) - new_time_mark++ * 1000);
-                        $.newChapterCnt++;
-                    }
-                    return {
-                        link: resolve('https://www.mhgui.com/', a.attr('href')),
-                        title: a.attr('title'),
-                        pub_date: pDate,
-                        num: a.find('i').text(),
-                        guid: resolve('https://www.mhgui.com/', a.attr('href')),
-                        category: categoryName,
-                    };
-                });
-        })
-        .reduce((acc, curr) => acc.concat(curr));
-};
-module.exports = async (ctx) => {
-    const { id } = ctx.params;
-    const chapterCnt = Number(ctx.params.chapterCnt || 0);
-    const { data } = await got.get(`https://www.mhgui.com/comic/${id}/`);
-    const $ = cheerio.load(data);
-
-    if ($('#__VIEWSTATE').length > 0) {
-        const n = LZString.decompressFromBase64($('#__VIEWSTATE').val());
-        if (n) {
-            $('#erroraudit_show').replaceWith(n);
-            $('#__VIEWSTATE').remove();
-        }
-    }
-    const bookTitle = $('.book-title > h1').text();
-    const bookIntro = $('#intro-all').text();
-    const coverImgSrc = $('.book-cover img').attr('src');
-    // 对最新更新的章节增加了pubDate
-    const reg = new RegExp('最近于.+更新至');
-    // 处理已下架的漫画
-    if ($('.status > span').text().indexOf('已下架') > 0) {
-        ctx.state.data = {
-            title: `看漫画 - ${bookTitle} 已下架`,
-            link: `https://www.mhgui.com/comic/${id}/`,
-            description: bookIntro,
-            item: [{ link: `https://www.mhgui.com/comic/${id}/`, title: bookTitle, description: '已下架' }],
-        };
-    } else {
-        const pub_date_str = $('.status > span').text().match(reg)[0].replace('最近于 [', '').replace('] 更新至', '');
-        // 为了能在闭包内访问到这个日期而不是每次需要处理这个最近更新日期
-        $.pubDate = new Date(pub_date_str).toUTCString();
-        $.newChapterCnt = 0;
-        const chapters = getChapters($);
-        const genResult = (chapter) => ({
-            link: chapter.link,
-            title: chapter.title,
-            pubDate: chapter.pub_date,
-            guid: chapter.guid,
-            category: chapter.category,
-            description: `
-            <h1>${chapter.num}</h1>
-            <img src='${coverImgSrc}' />
-        `.trim(),
-        });
-        const items = chapters.map(genResult);
-        let itemsLen = items.length;
-        if (chapterCnt > 0) {
-            itemsLen = chapterCnt < $.newChapterCnt ? $.newChapterCnt : chapterCnt;
-        }
-        ctx.state.data = {
-            title: `看漫画 - ${bookTitle}`,
-            link: `https://www.mhgui.com/comic/${id}/`,
-            description: bookIntro,
-            item: items.slice(0, itemsLen),
-        };
-    }
-};

package/lib/routes/mofcom/article.js

@@ -1,54 +0,0 @@
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-const { resolve } = require('url');
-
-const host = 'http://www.mofcom.gov.cn';
-
-module.exports = async (ctx) => {
-    const suffix = ctx.params.suffix;
-    const url = resolve('http://www.mofcom.gov.cn/article/', suffix);
-    const res = await got.get(url);
-    const $ = cheerio.load(res.data);
-    const title = $('head > title').text();
-    const list = $('.txtList_01 li').get();
-
-    const out = await Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const title = $('a').attr('title');
-            const sub_url = $('a').attr('href');
-            const itemUrl = resolve(host, sub_url);
-            const cache = await ctx.cache.get(itemUrl);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            let responses = await got.get(itemUrl);
-            const redirect = responses.data.match(/window\.location\.href='(.*)'/);
-            if (redirect) {
-                responses = await got.get(redirect[1], {
-                    headers: {
-                        Referer: itemUrl,
-                        'Accept-Encoding': 'gzip, deflate',
-                        'Accept-Language': 'zh-CN,zh;q=0.9',
-                        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3',
-                    },
-                });
-            }
-            const $d = cheerio.load(responses.data);
-
-            const single = {
-                title,
-                link: itemUrl,
-                description: $d('.art-con').html() || $d('.textlive').html(),
-            };
-            ctx.cache.set(itemUrl, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
-    );
-    ctx.state.data = {
-        title,
-        link: url,
-        item: out,
-    };
-};

package/lib/routes/nautilus/topics.js

@@ -1,45 +0,0 @@
-const cheerio = require('cheerio');
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    const url = `http://nautil.us/term/f/${ctx.params.tid}`;
-
-    const res = await got.get(url);
-    const $ = cheerio.load(res.data);
-    const list = $('article.search-result').get();
-
-    const out = await Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const title = $('h3 > a').text();
-            const partial = $('h3 > a').attr('href');
-            const address = `http://nautil.us${partial}`;
-            const cache = await ctx.cache.get(address);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-            const res = await got.get(address);
-            const capture = cheerio.load(res.data);
-            capture('div.reco').remove();
-            const banner = capture('div.banner').html();
-            const textWithTime = capture('div.byline').text();
-            const author = capture('div.byline > span:nth-child(2)').text();
-            const contents = banner + capture('div.page-content').html();
-            const single = {
-                title,
-                author,
-                description: contents,
-                link: address,
-                guid: address,
-                pubDate: new Date(textWithTime).toUTCString(),
-            };
-            ctx.cache.set(address, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
-    );
-    ctx.state.data = {
-        title: 'Nautilus | ' + $('title').text(),
-        link: url,
-        item: out,
-    };
-};