rsshub 1.0.0-master.f2ed98e → 1.0.0-master.f2eee10

package/lib/routes/gov/news/index.js

@@ -1,115 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const logger = require('@/utils/logger');
-
-module.exports = async (ctx) => {
-    const uid = ctx.params.uid;
-    let url = '';
-    let title = '';
-    let list = '';
-    switch (uid) {
-        case 'bm':
-            url = `http://www.gov.cn/xinwen/lianbo/bumen.htm`;
-            title = '中国政府网 - 部门政务';
-            break;
-        case 'yw':
-            url = `http://www.gov.cn/xinwen/yaowen.htm`;
-            title = '中国政府网 - 新闻要闻';
-            break;
-        case 'gd':
-            url = `http://www.gov.cn/xinwen/gundong.htm`;
-            title = '中国政府网 - 滚动新闻';
-            break;
-        case 'gwy':
-            url = 'http://www.gov.cn/pushinfo/v150203/';
-            title = '中国政府网 - 国务院信息';
-            break;
-        case 'zhengce':
-            url = 'http://sousuo.gov.cn/s.htm?t=zhengcelibrary';
-            title = '中国政府网 - 政策文件';
-            break;
-        default:
-            logger.error('pattern not matched');
-    }
-    const listData = await got.get(url);
-    const $ = cheerio.load(listData.data);
-    if (url.search(/zhengcelibrary/g) !== -1) {
-        list = $('.dys_middle_result_content_item');
-    } else {
-        list = $('.news_box .list li:not(.line)');
-    }
-
-    ctx.state.data = {
-        title,
-        link: url,
-        item: await Promise.all(
-            list
-                .slice(0, 12)
-                .map(async (index, item) => {
-                    item = $(item);
-                    let fullText = '';
-                    let href = '';
-                    let contentUrl = '';
-                    let description = '';
-                    let fullTextData = '';
-                    let fullTextGet = '';
-                    if (item.html().search(/dysMiddleResultConItemTitle/g) !== -1) {
-                        contentUrl = item.find('a').attr('href');
-                        if (contentUrl.search(/content/g) === -1) {
-                            description = item.find('a').text(); // 忽略获取吹风会的全文
-                        } else {
-                            description = await ctx.cache.tryGet(contentUrl, async () => {
-                                fullTextGet = await got.get(contentUrl);
-                                fullTextData = cheerio.load(fullTextGet.data);
-                                if (fullTextData.html().search(/pages_content/g) !== -1) {
-                                    fullTextData('.shuzi').remove(); // 移除videobg的图片
-                                    fullTextData('#myFlash').remove(); // 移除flash
-                                    return fullTextData('.pages_content').html();
-                                } else {
-                                    fullTextData('.shuzi').remove(); // 移除videobg的图片
-                                    fullTextData('#myFlash').remove(); // 移除flash
-                                    return fullTextData('#UCAP-CONTENT').html();
-                                }
-                            });
-                        }
-                    } else {
-                        href = item.find('a').attr('href');
-                        contentUrl = href.startsWith('/') ? `http://www.gov.cn${href}` : href;
-                        if (contentUrl.search(/content/g) === -1) {
-                            description = item.find('a').text(); // 忽略获取吹风会的全文
-                        } else {
-                            description = await ctx.cache.tryGet(contentUrl, async () => {
-                                fullTextGet = await got.get(contentUrl);
-                                fullTextData = cheerio.load(fullTextGet.data);
-                                const $1 = fullTextData.html();
-                                if (contentUrl.search(/zhengceku/g) !== -1) {
-                                    // 政策文件库
-                                    fullText = fullTextData('.pages_content').html();
-                                } else {
-                                    if ($1.search(/UCAP-CONTENT/g) === -1) {
-                                        fullTextData('.shuzi').remove(); // 移除videobg的图片
-                                        fullTextData('#myFlash').remove(); // 移除flash
-                                        fullText = fullTextData('body').html();
-                                    } else {
-                                        fullTextData('.shuzi').remove(); // 移除videobg的图片
-                                        fullTextData('#myFlash').remove(); // 移除flash
-                                        fullText = fullTextData('#UCAP-CONTENT').html();
-                                    }
-                                }
-                                const regImg = /(img src=")(.*)(".*)/g;
-                                const regUrl = /(http.*\/)(content.*)/;
-                                const urlSplit = contentUrl.replace(regUrl, '$1');
-                                return fullText.replace(regImg, '$1' + urlSplit + '$2' + '$3'); // 处理图片链接
-                            });
-                        }
-                    }
-                    return {
-                        title: item.find('a').text(),
-                        description,
-                        link: contentUrl,
-                    };
-                })
-                .get()
-        ),
-    };
-};

package/lib/routes/hinatazaka46/blog.js

@@ -1,38 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: 'https://www.hinatazaka46.com/s/official/diary/member',
-        headers: {
-            Referer: 'http://www.hinatazaka46.com/',
-        },
-    });
-
-    const data = response.data;
-    const $ = cheerio.load(data);
-    const list = $('div.p-blog-top__contents ul li');
-    let itemPicUrl;
-
-    ctx.state.data = {
-        allowEmpty: true,
-        title: '日向坂46官网 博客',
-        link: 'https://www.hinatazaka46.com/s/official/diary/member',
-        item:
-            list &&
-            list
-                .map((index, item) => {
-                    item = $(item);
-                    itemPicUrl = item.find('div.c-blog__image').attr('style').replace('background-image:url(', '').replace(');', '');
-                    return {
-                        title: item.find('p.c-blog-top__title').text().trim(),
-                        link: item.find('a').first().attr('href'),
-                        pubDate: item.find('time.c-blog-top__date').text(),
-                        author: item.find('div.c-blog-top__name').text().trim(),
-                        description: `<img src="${itemPicUrl}">`,
-                    };
-                })
-                .get(),
-    };
-};

package/lib/routes/hinatazaka46/news.js

@@ -1,34 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
-        url: 'https://www.hinatazaka46.com/s/official/news/list',
-        headers: {
-            Referer: 'http://www.hinatazaka46.com/',
-        },
-    });
-
-    const data = response.data;
-    const $ = cheerio.load(data);
-    const list = $('main div.l-maincontents--news ul.p-news__list li');
-
-    ctx.state.data = {
-        allowEmpty: true,
-        title: '日向坂46官网 NEWS',
-        link: 'http://www.hinatazaka46.com/news/',
-        item:
-            list &&
-            list
-                .map((index, item) => {
-                    item = $(item);
-                    return {
-                        title: item.find('a p.c-news__text').first().text(),
-                        link: item.find('a').attr('href'),
-                        pubDate: item.find('a time.c-news__date').first().text(),
-                    };
-                })
-                .get(),
-    };
-};

package/lib/routes/icable/category.js

@@ -1,64 +0,0 @@
-const got = require('@/utils/got');
-const timezone = require('@/utils/timezone');
-
-function desc(item, option) {
-    if (option === 'withphoto') {
-        return `<img src="${item.pic_url}"><p>${item.desc}</p>`;
-    } else if (option === 'plain') {
-        return item.desc;
-    } else {
-        throw Error(`Invalid option ‘<code>${option}</code>’. Please refer to <a href="https://docs.rsshub.app/traditional-media.html#i-cable-you-xian-xin-wen">the documention</a>.`);
-    }
-}
-
-module.exports = async (ctx) => {
-    const category = ctx.params.category;
-    const option = ctx.params.option || 'withphoto';
-
-    const response = await got({
-        method: 'get',
-        url: `http://cablenews.i-cable.com/ci/news/listing/api`,
-        headers: {
-            Referer: `http://cablenews.i-cable.com/ci/news/listing`,
-        },
-    });
-
-    const data = response.data;
-    let result;
-    let feedTitle;
-    if (category === 'all') {
-        result = data;
-        feedTitle = 'i-CABLE 即時新聞';
-    } else if (category === 'local') {
-        result = data.filter((w) => w.group_id === '128');
-        feedTitle = '港聞 - i-CABLE 即時新聞';
-    } else if (category === 'international') {
-        result = data.filter((w) => w.group_id === '136');
-        feedTitle = '兩岸國際 - i-CABLE 即時新聞';
-    } else if (category === 'china') {
-        result = data.filter((w) => w.group_id === '18');
-        feedTitle = '有線中國組 - i-CABLE 即時新聞';
-    } else if (category === 'sports') {
-        result = data.filter((w) => w.group_id === '321');
-        feedTitle = '體育 - i-CABLE 即時新聞';
-    } else {
-        throw Error(`Invalid category ‘<code>${category}</code>’. Please refer to <a href="https://docs.rsshub.app/traditional-media.html#i-cable-you-xian-xin-wen">the documention</a>.`);
-    }
-
-    const items = result.map((item) => ({
-        title: item.title,
-        author: '有線新聞',
-        category: item.group_name,
-        description: desc(item, option),
-        pubDate: timezone(new Date(String(item.created_at)), +8),
-        guid: item.id,
-        link: `http://cablenews.i-cable.com/ci/news/article/37/${item.id}`,
-    }));
-
-    ctx.state.data = {
-        title: feedTitle,
-        link: 'http://cablenews.i-cable.com/ci/news/listing',
-        description: '有線新聞 - 走在事實最前線',
-        item: items,
-    };
-};

package/lib/routes/instagram/index.js

@@ -1,131 +0,0 @@
-const cheerio = require('cheerio');
-const { IgLoginRequiredError } = require('instagram-private-api');
-const { ig, login } = require('./utils');
-const logger = require('@/utils/logger');
-const config = require('@/config').value;
-
-// loadContent pulls the desired user/tag/etc
-async function loadContent(category, nameOrId) {
-    let title, link, description, logo;
-    let itemsRaw;
-
-    // TODO: extend to other feed categories
-    switch (category) {
-        case 'user': {
-            let userInfo, username, id;
-            if (!isNaN(nameOrId)) {
-                id = nameOrId;
-                userInfo = await ig.user.info(id);
-                username = userInfo.username;
-            } else {
-                username = nameOrId;
-                id = await ig.user.getIdByUsername(username);
-                userInfo = await ig.user.info(id);
-            }
-
-            description = userInfo.biography;
-            logo = userInfo.profile_pic_url;
-            const fullName = userInfo.full_name;
-            title = `${fullName} (@${username}) - Instagram`;
-            link = `https://www.instagram.com/${username}`;
-
-            itemsRaw = await ig.feed.user(id).items();
-            break;
-        }
-        case 'tag': {
-            const tag = nameOrId;
-
-            title = `#${tag}) - Instagram`;
-            link = `https://www.instagram.com/explore/tags/${tag}`;
-
-            itemsRaw = await (await ig.feed.tags(tag).items()).slice(1); // the 0 item is undefined for unknown reason
-            break;
-        }
-        default: {
-            break;
-        }
-    }
-
-    return {
-        title,
-        link,
-        description,
-        logo,
-        itemsRaw,
-    };
-}
-
-module.exports = async (ctx) => {
-    // https://github.com/dilame/instagram-private-api#feeds
-    // const availableCategories = ["accountFollowers", "accountFollowing", "news",
-    //     "discover", "pendingFriendships", "blockedUsers", "directInbox", "directPending",
-    //     "directThread", "user", "tag", "location", "mediaComments", "reelsMedia", "reelsTray",
-    //     "timeline", "musicTrending", "musicSearch", "musicGenre", "musicMood", "usertags", "saved"];
-    const availableCategories = ['user', 'tag'];
-    const category = ctx.params.category;
-    if (!availableCategories.includes(category)) {
-        throw Error('Such feed is not supported.');
-    }
-
-    if (config.instagram && config.instagram.proxy) {
-        ig.state.proxyUrl = config.instagram.proxy;
-    }
-
-    // Unique key for a feed category
-    // e.g. username for user feed
-    const key = ctx.params.key;
-
-    let data;
-    try {
-        data = await loadContent(category, key);
-    } catch (e) {
-        logger.error(`Instagram error: ${e}`);
-        if (e instanceof IgLoginRequiredError) {
-            // Login and try again
-            await login(ig);
-            data = await loadContent(category, key);
-        } else {
-            throw e;
-        }
-    }
-
-    const items = data.itemsRaw.map((item) => {
-        // Content
-        const summary = item.caption ? item.caption.text : '';
-        const $ = cheerio.load(`<p>${summary}</p>`);
-        const content = $.root();
-
-        // Image or carousel media
-        const images = item.image_versions2 ? [item.image_versions2] : item.carousel_media.map((i) => i.image_versions2);
-        for (const i of images) {
-            const img = i.candidates[0];
-            content.append(`<img src="${img.url}" height="${img.height}" width="${img.width}">`);
-        }
-
-        // Metadata
-        const url = `https://www.instagram.com/p/${item.code}`;
-        const pubDate = new Date(item.taken_at * 1000).toUTCString();
-        const title = summary.split('\n')[0];
-
-        return {
-            title,
-            id: item.pk,
-            pubDate,
-            author: item.creator,
-            link: url,
-            summary,
-            description: content.html(),
-            icon: 'https://www.instagram.com/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png',
-            logo: data.logo,
-        };
-    });
-
-    ctx.state.data = {
-        title: data.title,
-        link: data.link,
-        description: data.description,
-        item: items,
-        icon: 'https://www.instagram.com/static/images/ico/xxhdpi_launcher.png/99cf3909d459.png',
-        logo: data.logo,
-    };
-};

package/lib/routes/instagram/utils.js

@@ -1,28 +0,0 @@
-const { IgApiClient } = require('instagram-private-api');
-const logger = require('@/utils/logger');
-const config = require('@/config').value;
-
-const ig = new IgApiClient();
-() => login(ig); // deepscan-disable-line UNUSED_EXPR
-
-async function login(ig) {
-    if (config.instagram && config.instagram.username && config.instagram.password) {
-        try {
-            ig.state.generateDevice(config.instagram.username);
-            try {
-                await ig.simulate.preLoginFlow();
-            } catch (error) {
-                logger.info('Instagram preLoginFlow fail: ' + error);
-            }
-            await ig.account.login(config.instagram.username, config.instagram.password);
-            process.nextTick(() => ig.simulate.postLoginFlow());
-            logger.info('Instagram login success.');
-        } catch (error) {
-            logger.error('Instagram login fail: ' + error);
-        }
-    } else {
-        throw Error('Instagram RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>');
-    }
-}
-
-module.exports = { ig, login };

package/lib/routes/maitta/index.js

@@ -1,29 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const url = 'https://www.ibc.co.jp/radio/maitta/audio/';
-
-module.exports = async (ctx) => {
-    const res = await got.get(url);
-    const $ = cheerio.load(res.data);
-    const items = $('.broadcast').get();
-    ctx.state.data = {
-        title: 'IBCラジオ　イヤーマイッタマイッタ｜IBC岩手放送',
-        link: 'http://www.ibc.co.jp/radio/maitta/audio',
-        description: $('meta[name=description]').attr('content'),
-        itunes_author: '水越アナと大塚アナ',
-        image: 'https://cdn.ibc.co.jp/radio/maitta/audio/images/og.png',
-        language: 'ja',
-        item: items.map((item) => {
-            item = $(item);
-            return {
-                title: item.find('h3').text(),
-                description: item.find('.linecontent').text().trim(),
-                link: `https:${item.find('a').first().attr('href').split('?')[0]}`,
-                pubDate: new Date(item.find('.onairdate').text().split('日')[0].replace(/年|月/g, '-')).toUTCString(),
-                itunes_item_image: 'https://cdn.ibc.co.jp/radio/maitta/audio/images/og.png',
-                enclosure_url: `https:${item.find('a').first().attr('href').split('?')[0]}`, // 音频链接
-                enclosure_type: 'audio/mpeg',
-            };
-        }),
-    };
-};

package/lib/routes/mastodon/timeline_local.js

@@ -1,22 +0,0 @@
-const got = require('@/utils/got');
-const utils = require('./utils');
-const config = require('@/config').value;
-
-module.exports = async (ctx) => {
-    const site = ctx.params.site;
-    const only_media = ctx.params.only_media ? 'true' : 'false';
-    if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) {
-        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
-    }
-
-    const url = `http://${site}/api/v1/timelines/public?local=true&only_media=${only_media}`;
-
-    const response = await got.get(url);
-    const list = response.data;
-
-    ctx.state.data = {
-        title: `Local Public${ctx.params.only_media ? ' Media' : ''} Timeline on ${site}`,
-        link: `http://${site}`,
-        item: utils.parseStatuses(list),
-    };
-};

package/lib/routes/mastodon/timeline_remote.js

@@ -1,22 +0,0 @@
-const got = require('@/utils/got');
-const utils = require('./utils');
-const config = require('@/config').value;
-
-module.exports = async (ctx) => {
-    const site = ctx.params.site;
-    const only_media = ctx.params.only_media ? 'true' : 'false';
-    if (!config.feature.allow_user_supply_unsafe_domain && !utils.allowSiteList.includes(site)) {
-        ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
-    }
-
-    const url = `http://${site}/api/v1/timelines/public?remote=true&only_media=${only_media}`;
-
-    const response = await got.get(url);
-    const list = response.data;
-
-    ctx.state.data = {
-        title: `Federated Public${ctx.params.only_media ? ' Media' : ''} Timeline on ${site}`,
-        link: `http://${site}`,
-        item: utils.parseStatuses(list),
-    };
-};

package/lib/routes/mastodon/utils.js

@@ -1,131 +0,0 @@
-const got = require('@/utils/got');
-const { parseDate } = require('@/utils/parse-date');
-
-const allowSiteList = ['mastodon.social', 'pawoo.net'];
-
-const parseStatuses = (data) =>
-    data.map((item) => {
-        // docs on: https://docs.joinmastodon.org/entities/status/
-
-        const accountRepostedBy = item.reblog ? item.account : null;
-        item = item.reblog ? item.reblog : item;
-
-        const content = item.content ? item.content.replace(/<span.*?>|<\/span.*?>/gm, '') : '';
-        const contentRemovedHtml = content.replace(/<(?:.|\n)*?>/gm, '\n');
-
-        const mediaParse = (media_attachments) =>
-            media_attachments
-                .map((item) => {
-                    const selectedUrl = item.remote_url ? item.remote_url : item.url;
-                    const description = item.description ? item.description : '';
-                    switch (item.type) {
-                        case 'gifv':
-                            return `<br><video src="${selectedUrl}" autoplay loop>gif ${description}</video>`;
-                        case 'video':
-                            return `<br><video src="${selectedUrl}" controls loop>video ${description}</video>`;
-                        case 'image':
-                            return `<br><img src="${selectedUrl}" alt="image ${description}">`;
-                        case 'audio':
-                            return `<br><audio controls src="${selectedUrl}">audio ${description}</audio>`;
-                        case 'unknown':
-                        default:
-                            return `<br><a href="${selectedUrl}">media ${description}</a>`;
-                    }
-                })
-                .join('');
-
-        const author = `${item.account.display_name} (@${item.account.acct})`;
-        const link = item.url;
-        const media = mediaParse(item.media_attachments);
-
-        const titleAuthor = accountRepostedBy ? `Re @${accountRepostedBy.username}` : `@${item.account.username}`;
-        const titleText = item.sensitive === true ? `(CW) ${item.spoiler_text}` : contentRemovedHtml;
-        const title = `${titleAuthor}: "${titleText}"`;
-
-        return {
-            title,
-            author,
-            description: item.spoiler_text + '<hr />' + content + media,
-            pubDate: parseDate(item.created_at),
-            link,
-            guid: item.uri,
-        };
-    });
-
-async function getAccountStatuses(site, account_id, only_media) {
-    const statuses_url = `http://${site}/api/v1/accounts/${account_id}/statuses?only_media=${only_media}`;
-    const statuses_response = await got({
-        method: 'get',
-        url: statuses_url,
-    });
-    const data = statuses_response.data;
-
-    let account_data;
-    if (data.length !== 0 && data[0].account !== null) {
-        account_data = data[0].account;
-    } else {
-        const account_url = `http://${site}/api/v1/accounts/${account_id}`;
-        const account_response = await got({
-            method: 'get',
-            url: account_url,
-        });
-        account_data = account_response.data;
-    }
-
-    return { account_data, data };
-}
-
-async function getAccountIdByAcct(acct, ctx) {
-    const config = require('@/config').value;
-    const mastodonConfig = config.mastodon;
-
-    if (!(mastodonConfig.apiHost && mastodonConfig.accessToken && mastodonConfig.acctDomain)) {
-        throw 'this route require api configuration, please check <a href="https://docs.rsshub.app/en/install/#configuration">documentation</a>';
-    }
-
-    const site = mastodonConfig.apiHost;
-
-    const search_url = `http://${site}/api/v2/search`;
-    const cacheUid = `mastodon_acct_id/${site}/${acct}`;
-
-    const account_id = await ctx.cache.tryGet(cacheUid, async () => {
-        const search_response = await got({
-            method: 'get',
-            url: search_url,
-            headers: {
-                Authorization: `Bearer ${mastodonConfig.accessToken}`,
-            },
-            searchParams: {
-                q: acct,
-                type: 'accounts',
-            },
-        });
-        const [acctUser, acctHost] = acct.split('@').filter((e) => e);
-        let acctOnServer;
-
-        if (acctHost) {
-            if (acctHost === mastodonConfig.acctDomain) {
-                acctOnServer = acctUser;
-            } else {
-                acctOnServer = acctUser + '@' + acctHost;
-            }
-        } else {
-            acctOnServer = acctUser;
-        }
-
-        const accountData = search_response.data.accounts.filter((item) => item.acct === acctOnServer);
-
-        if (accountData.length === 0) {
-            throw `acct ${acct} not found`;
-        }
-        return accountData[0].id;
-    });
-    return { site, account_id };
-}
-
-module.exports = {
-    parseStatuses,
-    getAccountStatuses,
-    getAccountIdByAcct,
-    allowSiteList,
-};