rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f812e91

package/lib/v2/gq/tw/index.js

@@ -1,165 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const md = require('markdown-it')({
-    linkify: true,
-});
-const { art } = require('@/utils/render');
-const path = require('path');
-
-const baseUrl = 'https://www.gq.com.tw';
-const parsePreloadedStateJSON = ($) =>
-    JSON.parse(
-        $('script[type="text/javascript"]')
-            .text()
-            .match(/window\.__PRELOADED_STATE__ = ({.*?});/)[1]
-    );
-const largestImage = (sources, id) => {
-    if (!id) {
-        let maxWidth = 0;
-        let maxUrl = '';
-        Object.values(sources).forEach((size) => {
-            if (size.width > maxWidth) {
-                maxWidth = size.width;
-                maxUrl = size.url;
-            }
-        });
-        return maxUrl;
-    }
-    const url = sources[Object.keys(sources)[0]].url;
-    const filename = url.substring(url.lastIndexOf('/') + 1);
-    return `https://media.gq.com.tw/photos/${id}/${filename}`;
-};
-
-module.exports = async (ctx) => {
-    const { caty, subcaty } = ctx.params;
-    const link = `${baseUrl}${caty ? `/${caty}` : ''}${subcaty ? `/${subcaty}` : ''}`;
-    const { data: response } = await got(link);
-    const $ = cheerio.load(response);
-    const { transformed } = parsePreloadedStateJSON($);
-
-    const list = transformed.bundle.containers
-        .filter((item) => item.items)
-        .map((item) =>
-            item.items.map((item) => ({
-                title: item.source?.hed || item.dangerousHed,
-                description: item.source?.dek || item.dangerousDek,
-                link: (item.url.startsWith('http') ? item.url : `${baseUrl}${item.url}`).split('#intcid')[0],
-                pubDate: parseDate(item.pubDate),
-                author: item.contributors.author.items.map((item) => item.name).join(', '),
-            }))
-        )
-        .flat();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const { data: response } = await got(item.link);
-                const $ = cheerio.load(response);
-                const data = JSON.parse($('script[type="application/ld+json"]').first().text());
-                const { transformed } = parsePreloadedStateJSON($);
-                const VideoObject = data['@type'] === 'VideoObject';
-                const lede = !VideoObject ? transformed.article.headerProps.lede : null;
-
-                let articleBody = '';
-                if (!VideoObject) {
-                    // typical article
-                    // articleBody inludes non standard markdown syntax
-                    articleBody = cheerio.load(md.render(data.articleBody.replace(/\{: target="_blank"\}/g, '')).replace(/\{: #link\}/g, ''), null, false);
-                    articleBody('a').each((_, item) => {
-                        item = $(item);
-                        if (item.text().startsWith('#video:')) {
-                            if (item.text().match(/youtube/)) {
-                                const videoId = item.text().match(/.*https:\/\/www\.youtube\.com\/embed\/(.*)\s/)[1];
-                                item.replaceWith(art(path.join(__dirname, '../templates/youtube.art'), { videoId }));
-                            }
-                        }
-                        if (item.text().startsWith('![#image:')) {
-                            const imageId = item.text().match(/!\[#image: \/photos\/(.*?)\]/)[1];
-                            const imgInfo = transformed.article.lightboxImages.find((item) => item.id === imageId);
-                            item.html(
-                                art(path.join(__dirname, '../templates/img.art'), {
-                                    src: largestImage(imgInfo.sources, imageId),
-                                    alt: imgInfo.dangerousCaption,
-                                })
-                            );
-                        }
-                    });
-                    articleBody('p').each((_, item) => {
-                        item = $(item);
-                        if (item.text().startsWith('+++')) {
-                            item.remove();
-                        }
-                        if (item.text().startsWith('[#image:')) {
-                            const imageId = item.text().match(/\[#image: \/photos\/(.*?)\]/)[1];
-                            const imgInfo = transformed.article.lightboxImages.find((item) => item.id === imageId);
-                            item.replaceWith(
-                                art(path.join(__dirname, '../templates/img.art'), {
-                                    src: largestImage(imgInfo.sources, imageId),
-                                    alt: imgInfo.dangerousCaption,
-                                })
-                            );
-                        }
-                        if (item.text().startsWith('[#article:')) {
-                            const articleId = item.text().match(/\[#article: \/articles\/(.*?)\]/)[1];
-                            const articleProps = transformed.article.body.filter((i) => i[0] === 'inline-embed').find((i) => i[1].ref === articleId)[1].props;
-                            item.replaceWith(
-                                art(path.join(__dirname, '../templates/embed-article.art'), {
-                                    url: `${baseUrl}${articleProps.url}`,
-                                    text: articleProps.dangerousHed,
-                                })
-                            );
-                        }
-                        if (item.text().startsWith('[#product:')) {
-                            const productId = item.text().match(/\[#product: \/products\/(.*?)\]/)[1];
-                            const productProps = transformed.article.body.filter((i) => i[0] === 'inline-embed').find((i) => i[1].ref === productId)[1].props;
-                            item.replaceWith(
-                                art(path.join(__dirname, '../templates/embed-product.art'), {
-                                    img: art(path.join(__dirname, '../templates/img.art'), {
-                                        src: largestImage(productProps.image.sources),
-                                    }),
-                                    productProps,
-                                })
-                            );
-                        }
-                        if (item.text().startsWith('[#instagram:')) {
-                            const instagramHref = item.text().match(/\[#instagram: (.*?)\]/)[1];
-                            item.replaceWith(
-                                art(path.join(__dirname, '../templates/embed-article.art'), {
-                                    url: instagramHref,
-                                    text: instagramHref,
-                                })
-                            );
-                        }
-                    });
-                    item.description = art(path.join(__dirname, '../templates/tw.art'), {
-                        dangerousDek: transformed.article.headerProps.dangerousDek, // quote
-                        lede: art(path.join(__dirname, '../templates/img.art'), {
-                            // header image
-                            src: lede?.sources ? largestImage(lede.sources, lede.id) : null,
-                            alt: lede?.caption,
-                        }),
-                        articleBody: articleBody.html(),
-                    });
-                } else {
-                    // is VideoObject
-                    item.description = art(path.join(__dirname, '../templates/videoObject.art'), {
-                        poster: transformed.video.metaImageUrl,
-                        sources: transformed.video.sources,
-                        articleBody: data.description,
-                    });
-                }
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: `${transformed.coreDataLayer.content.contentTitle} | ${transformed.coreDataLayer.content.brand}`,
-        link,
-        image: `${baseUrl}${transformed.logo.sources.sm.url}`,
-        item: items,
-        allowEmpty: true,
-    };
-};

package/lib/v2/guandian/index.js

@@ -1,37 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
-
-const baseUrl = 'https://www.guandian.cn';
-
-module.exports = async (ctx) => {
-    const cat = ctx.params.category;
-    const reqUrl = `${baseUrl}/${cat}/`;
-    const response = await got(reqUrl);
-    const $ = cheerio.load(response.data);
-
-    const items = $('.con_l li')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-            const a = item.find('.con_l_con_r a');
-            const link = a.attr('href');
-            const title = a.find('h3').text();
-            const desc = item.find('.con_l_con_r p').text();
-
-            const dateStr = item.find('#keyword span').text();
-            return {
-                title,
-                link,
-                description: desc,
-                pubDate: timezone(parseDate(dateStr), 8),
-            };
-        });
-
-    ctx.state.data = {
-        title: $('head title').text(),
-        link: reqUrl,
-        item: items,
-    };
-};

package/lib/v2/guandian/maintainer.js

@@ -1,3 +0,0 @@
-module.exports = {
-    '/:category': ['drgnchan'],
-};

package/lib/v2/guandian/radar.js

@@ -1,13 +0,0 @@
-module.exports = {
-    'guandian.cn': {
-        _name: '观点网',
-        www: [
-            {
-                title: '资讯',
-                docs: 'https://docs.rsshub.app/routes/new-media#guan-dian-wang-zi-xun',
-                source: ['/:category'],
-                target: '/guandian/:category',
-            },
-        ],
-    },
-};

package/lib/v2/guandian/router.js

@@ -1,3 +0,0 @@
-module.exports = function (router) {
-    router.get('/:category', require('./index'));
-};

package/lib/v2/gz-cmc/index.js

@@ -1,137 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const timezone = require('@/utils/timezone');
-const { parseDate } = require('@/utils/parse-date');
-const { art } = require('@/utils/render');
-const path = require('path');
-
-module.exports = async (ctx) => {
-    let channel = '';
-    let site = '';
-    let siteName = '广州市融媒体中心';
-    let siteLink = 'https://www.gz-cmc.com/';
-    switch (ctx.params.site) {
-        case 'huacheng':
-        case 'xinhuacheng':
-        case 'hc':
-        case 'guangzhou':
-        case 'gz':
-        case '':
-            channel = ctx.params.channel ?? 'shouye';
-            site = 'huacheng';
-            siteName = '广州日报——新花城';
-            siteLink = `https://huacheng.gz-cmc.com/channel/${channel}/index.html`;
-            break;
-
-        case 'daohuangpuqu':
-        case 'huangpu':
-        case 'hp':
-            channel = ctx.params.channel ?? 'sy';
-            if (channel === 'wanqushibao' || channel === 'wqsb') {
-                channel = 'hpxsd';
-            }
-            site = 'daohuangpuqu';
-            siteName = '黄埔融媒——到黄埔去';
-            siteLink = 'https://www.gz-cmc.com/html/download.html?siteId=cd4f63bdfdfd41749c7e304bc0d7a6df'; // no website available for this site
-            break;
-
-        case 'zhangshangfanyu':
-        case 'zhangshangpanyu':
-        case 'panyu':
-        case 'py':
-            channel = ctx.params.channel ?? 'shouye';
-            site = 'zhangshangfanyu'; // not a typo here...
-            siteName = '番禺融媒——掌上番禺';
-            siteLink = 'https://www.gz-cmc.com/html/download.html?siteId=d755df8bb9fb49ee96052f7dee58d21b'; // no website available for this site
-            break;
-
-        case 'yuezengcheng':
-        case 'zengcheng':
-        case 'zc':
-            channel = ctx.params.channel ?? 'shouye';
-            site = 'yuezengcheng';
-            siteName = '增城融媒——阅增城';
-            siteLink = 'https://www.gz-cmc.com/html/download.html?siteId=78fa6d06b0dd4f27abf341e5efde035a'; // no website available for this site
-            break;
-
-        case 'guangzhoubaiyun':
-        case 'baiyun':
-        case 'by':
-            channel = ctx.params.channel ?? 'sy';
-            site = 'guangzhoubaiyun';
-            siteName = '白云融媒——广州白云';
-            siteLink = `https://guangzhoubaiyun.gz-cmc.com/channel/${channel}/index.html`;
-            break;
-
-        default:
-            channel = ctx.params.channel;
-            site = ctx.params.site;
-            break;
-    }
-    const apiUrl = `https://${site}.gz-cmc.com/json/channel/${channel}/list.json`;
-
-    const response = await got({
-        method: 'get',
-        url: apiUrl,
-    });
-
-    let result = [];
-    if (site === 'daohuangpuqu' && channel === 'hpxsd') {
-        const { data: newResponse } = await got(response.data.list[0].data.detailJsonPath);
-        result = newResponse.dataList;
-        if (newResponse.nextJsonUrl) {
-            const { data: nextResponse } = await got(newResponse.nextJsonUrl);
-            result = [...result, ...nextResponse.dataList];
-        }
-    } else {
-        result = response.data.list;
-    }
-
-    const list = result
-        .filter((i) => i.data.contentType === 1 && i.data.linkType === 0)
-        .map((item) => ({
-            title: item.data.title,
-            description: art(path.join(__dirname, 'templates/description.art'), {
-                thumb: item.data.mCoverImg,
-            }),
-            pubDate: timezone(parseDate(item.data.lastpublishTime), +8),
-            link: item.data.url,
-            author: item.data.author,
-        }));
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-
-                const content = cheerio.load(detailResponse.data);
-                content('.broadcast-container').remove();
-
-                if (content('meta[name="description"]').attr('content')) {
-                    item.description += '<blockquote>' + content('meta[name="description"]').attr('content') + '</blockquote>';
-                }
-                item.description += content('.article-source').html() ?? '';
-
-                if (site === 'daohuangpuqu') {
-                    const articleContent = content('script')
-                        .text()
-                        .match(/contentTxt ="(.*)";/);
-                    item.description += articleContent ? articleContent[1].replace(/\\/g, '') : '';
-                } else {
-                    item.description += content('#articleContent').html();
-                }
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: siteName,
-        link: siteLink,
-        item: items,
-    };
-};

package/lib/v2/gz-cmc/maintainer.js

@@ -1,3 +0,0 @@
-module.exports = {
-    '/:site/:channel?': ['TimWu007'],
-};

package/lib/v2/gz-cmc/radar.js

@@ -1,12 +0,0 @@
-module.exports = {
-    'gz-cmc.com': {
-        _name: '广州市融媒体中心',
-        '.': [
-            {
-                title: '频道',
-                docs: 'https://docs.rsshub.app/routes/traditional-media#guang-zhou-shi-rong-mei-ti-zhong-xin',
-                source: ['/'],
-            },
-        ],
-    },
-};

package/lib/v2/gz-cmc/router.js

@@ -1,3 +0,0 @@
-module.exports = (router) => {
-    router.get('/:site/:channel?', require('./index'));
-};

package/lib/v2/gz-cmc/templates/description.art

@@ -1,3 +0,0 @@
-{{ if thumb }}
-<img src="{{ thumb }}"><br>
-{{ /if }}

package/lib/v2/gzh360/_README

@@ -1 +0,0 @@
-Make sure you read lib/utils/wechat-mp.js before adding a new route.

package/lib/v2/gzh360/category.js

@@ -1,8 +0,0 @@
-const universal = require('./universal');
-
-module.exports = async (ctx) => {
-    const path = 'category';
-    const id = ctx.params.id ?? '';
-    const titleHeader = '公众号 360 - ';
-    await universal(ctx, path, id, titleHeader);
-};

package/lib/v2/gzh360/gzh.js

@@ -1,18 +0,0 @@
-const { mpIdEncode, mpIdDecode } = require('./utils');
-const universal = require('./universal');
-
-module.exports = async (ctx) => {
-    const path = 'gzh_articles';
-
-    let name = ctx.params.name;
-    let id = name;
-
-    // in order to support RSSHub Radar, we need to accept both name and id
-    try {
-        name = mpIdDecode(name);
-    } catch {
-        id = mpIdEncode(name);
-    }
-
-    await universal(ctx, path, id, '', name, true, true);
-};

package/lib/v2/gzh360/maintainer.js

@@ -1,4 +0,0 @@
-module.exports = {
-    '/category/:id?': ['Rongronggg9'],
-    '/gzh/:name?': ['Rongronggg9'],
-};

package/lib/v2/gzh360/radar.js

@@ -1,19 +0,0 @@
-module.exports = {
-    'gzh360.com': {
-        _name: '公众号360',
-        web: [
-            {
-                title: '公众号',
-                docs: 'https://docs.rsshub.app/routes/new-media#gong-zhong-hao-360',
-                source: ['/gzh_articles', '/gzh', '/'],
-                target: (params, url) => `/gzh360/gzh/${new URL(url).searchParams.get('id') ?? ''}`,
-            },
-            {
-                title: '分类',
-                docs: 'https://docs.rsshub.app/routes/new-media#gong-zhong-hao-360',
-                source: ['/category', '/'],
-                target: (params, url) => `/gzh360/category/${new URL(url).searchParams.get('id') ?? ''}`,
-            },
-        ],
-    },
-};

package/lib/v2/gzh360/router.js

@@ -1,4 +0,0 @@
-module.exports = function (router) {
-    router.get('/category/:id?', require('./category'));
-    router.get('/gzh/:name', require('./gzh'));
-};

package/lib/v2/gzh360/universal.js

@@ -1,53 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const { finishArticleItem } = require('./utils');
-const config = require('@/config').value;
-
-const getInitEntry = (url) =>
-    got(url)
-        .then((_r) => _r.data)
-        .catch((err) => {
-            if (err.response.statusCode === 404) {
-                throw new Error('This category / WeChat Official Account is not found on GZH360.');
-            }
-            throw err;
-        });
-
-module.exports = async (ctx, path, id, titleHeader = '', custom_title = null, skipAuthor = false, cacheInitEntry = false) => {
-    const rootUrl = 'http://web.gzh360.com';
-    const currentUrl = `${rootUrl}/${id ? `${path}?id=${id}` : ''}`;
-
-    const respData = cacheInitEntry ? await ctx.cache.tryGet(currentUrl, () => getInitEntry(currentUrl), config.cache.routeExpire, false) : await getInitEntry(currentUrl);
-    const $ = cheerio.load(respData);
-
-    const title = id ? $('head > title').text().split(' - ', 1)[0] : '首页';
-
-    let items = $('div.content div.news_desc')
-        .map((_, item) => {
-            item = $(item);
-            const link = item.find('h3 > a');
-            const href = link.attr('href');
-            const title = link.text();
-            const pubDate = item.find('span.datecss > span').attr('data-timestamp');
-            let author;
-            if (!skipAuthor) {
-                author = item.find('span.fromcss > a').text(); // only some pages have this
-            }
-            return {
-                link: `${href.startsWith('http') ? '' : rootUrl}${href}`,
-                title,
-                pubDate: parseDate(pubDate),
-                author,
-            };
-        })
-        .get();
-
-    items = await Promise.all(items.map((item) => finishArticleItem(ctx, item, skipAuthor)));
-
-    ctx.state.data = {
-        title: `${titleHeader}${custom_title ?? title}`,
-        link: currentUrl,
-        item: items.filter((item) => item),
-    };
-};

package/lib/v2/gzh360/utils.js

@@ -1,62 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const { fixArticleContent } = require('@/utils/wechat-mp');
-
-const invalidIdError = new RangeError('Invalid id');
-
-const mpIdEncode = (name) => {
-    const onePassed = Buffer.from(name).toString('base64'); // one-pass uses the standard base64 alphabet
-    // two-passed base64 is always url-safe (`7f+/` never appear), you can prove it by yourself
-    return Buffer.from(onePassed).toString('base64');
-};
-
-const mpIdDecode = (id) => {
-    // verify that the decoded name can be a valid WeChat mp name: https://kf.qq.com/faq/120911VrYVrA141110r2MRJV.html
-    // "公众号名称/昵称可设置4-30个字符（1个汉字算2字符）": 4 ASCII characters => 12, 15 Chinese characters => 80
-    // and the id is a valid two-passed base64 string (`7f+/` never appear)
-    if (id.length < 12 || id.length > 80 || id.length % 4 !== 0 || !/^[a-eg-zA-Z0-68-9]+={0,2}$/.test(id)) {
-        throw invalidIdError;
-    }
-    const deSecondPassed = Buffer.from(id, 'base64').toString();
-    // verify that it is a valid base64 string using standard base64 alphabet
-    if (deSecondPassed.length % 4 !== 0 || !/^[a-zA-Z0-9+/]+={0,2}$/.test(deSecondPassed)) {
-        throw invalidIdError;
-    }
-    const deFirstPassed = Buffer.from(deSecondPassed, 'base64').toString();
-    // "空格不可在最前或者最后，且空格不可连续"
-    if (deFirstPassed.length < 2 || deFirstPassed.startsWith(' ') || deFirstPassed.endsWith(' ') || deFirstPassed.includes('  ')) {
-        throw invalidIdError;
-    }
-    return deFirstPassed;
-};
-
-const finishArticleItem = async (ctx, item, skipAuthor = false) => {
-    // the website is slow and unstable, so we need to ignore errors to avoid breaking the whole route
-    // we can't cache the item instead of the webpage because we need to retry when the last request failed
-    const article = await ctx.cache.tryGet(
-        item.link,
-        () =>
-            got(item.link)
-                .then((_r) => _r.data)
-                .catch(() => null) // it is safe do that in tryGet because a false value always lead to a cache miss
-    );
-
-    if (article) {
-        const $ = cheerio.load(article);
-        if (!skipAuthor) {
-            item.author = item.author || $('[id=webbdgzh]+a').text();
-        }
-        item.title = item.title || $('div.desc > h1').text();
-        item.pubDate = item.pubDate || parseDate($('div.desc span[data-timestamp]').attr('data-timestamp'));
-        item.description = fixArticleContent($('div.rich_media_content'), true); // sometimes it is an empty string due to the website's fault
-    }
-
-    return item;
-};
-
-module.exports = {
-    mpIdEncode,
-    mpIdDecode,
-    finishArticleItem,
-};

package/lib/v2/hellobtc/topic.js

@@ -1,43 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
-
-const rootUrl = 'https://www.hellobtc.com';
-
-module.exports = async (ctx) => {
-    const id = ctx.params.id;
-    const url = `${rootUrl}/topic/${id}.html`;
-
-    const response = await got(url);
-    const $ = cheerio.load(response.data);
-    const title = $('div.content > h1').text();
-    const list = $('div.tab_pane')
-        .find('div.new_item')
-        .map((_, item) => ({
-            title: $(item).find('h2').text(),
-            link: $(item).find('a').attr('href'),
-        }))
-        .filter((item) => item)
-        .get();
-
-    const items = await Promise.all(
-        list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got(item.link);
-                const content = cheerio.load(detailResponse.data);
-
-                item.description = content('#nr').html();
-                item.pubDate = timezone(parseDate(content('span.date').text(), 'YYYY-MM-DD HH:mm:ss'), +8);
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: `白话区块链 - ${title}专栏`,
-        link: url,
-        item: items,
-    };
-};

package/lib/v2/hjedd/article.js

@@ -1,27 +0,0 @@
-const got = require('@/utils/got');
-const { rootUrl, ProcessItems } = require('./utils');
-
-module.exports = async (ctx) => {
-    const node = ctx.params.node ?? '258';
-    const type = ctx.params.type ?? '1';
-
-    const currentUrl = rootUrl;
-    const nodeUrl = `${rootUrl}/api/topic/node?nodeId=${node}`;
-    const apiUrl = `${rootUrl}/api/topic/node/topics?page=1&nodeId=${node}&type=${type}&limit=${ctx.query.limit}`;
-
-    const response = await got({
-        method: 'get',
-        url: nodeUrl,
-    });
-
-    const data = response.data.data;
-
-    const items = await ProcessItems(apiUrl, ctx.query.limit, ctx.cache.tryGet);
-
-    ctx.state.data = {
-        title: `海角社区 - ${data.name}`,
-        link: `${currentUrl}/article?nodeId=${node}`,
-        item: items,
-        description: data.description.replace(/<\/?p>/, ''),
-    };
-};

package/lib/v2/hjedd/hot.js

@@ -1,14 +0,0 @@
-const { rootUrl, ProcessItems } = require('./utils');
-
-module.exports = async (ctx) => {
-    const currentUrl = rootUrl;
-    const apiUrl = `${rootUrl}/api/topic/hot/topics?page=1`;
-
-    const items = await ProcessItems(apiUrl, ctx.query.limit, ctx.cache.tryGet);
-
-    ctx.state.data = {
-        title: '海角社区 - 热门',
-        link: currentUrl,
-        item: items,
-    };
-};