rsshub 1.0.0-master.fea5824 → 1.0.0-master.feba548

package/lib/routes/theatlantic/utils.ts

@@ -0,0 +1,58 @@
+import { getCurrentPath } from '@/utils/helpers';
+const __dirname = getCurrentPath(import.meta.url);
+
+import cache from '@/utils/cache';
+import { load } from 'cheerio';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import * as path from 'node:path';
+import randUserAgent from '@/utils/rand-user-agent';
+
+const UA = randUserAgent({ browser: 'chrome', os: 'android', device: 'mobile' });
+
+const getArticleDetails = async (items) => {
+    const list = await Promise.all(
+        items.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const url = item.link;
+                const response = await got({
+                    url,
+                    method: 'get',
+                    headers: {
+                        'User-Agent': UA,
+                    },
+                });
+                const html = response.data;
+                const $ = load(html);
+                let data = JSON.parse($('script#__NEXT_DATA__').text());
+
+                const list = data.props.pageProps.urqlState;
+                const keyWithContent = Object.keys(list).filter((key) => list[key].data.includes('content'));
+                data = JSON.parse(list[keyWithContent].data).article;
+                item.title = data.shareTitle;
+                item.category = data.categories.map((category) => category.slug);
+                for (const channel of data.channels) {
+                    item.category.push(channel.slug);
+                }
+                item.content = data.content.filter((item) => item.innerHtml !== undefined && item.innerHtml !== '');
+                item.caption = data.dek;
+                item.imgUrl = data.leadArt.image?.url;
+                item.imgAlt = data.leadArt.image?.altText;
+                item.imgCaption = data.leadArt.image?.attributionText;
+                item.description = art(path.join(__dirname, 'templates/article-description.art'), {
+                    item,
+                });
+                return {
+                    title: item.title,
+                    pubDate: parseDate(item.pubDate),
+                    link: item.link,
+                    description: item.description,
+                };
+            })
+        )
+    );
+    return list;
+};
+
+export { getArticleDetails };

package/lib/routes/theblockbeats/index.ts

@@ -0,0 +1,54 @@
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+const domain = 'theblockbeats.info';
+const rootUrl = `https://www.${domain}`;
+const apiBase = `https://api.${domain}`;
+
+const channelMap = {
+    newsflash: {
+        title: '快讯',
+        link: `${rootUrl}/newsflash`,
+        api: `${apiBase}/v5/newsflash/select`,
+    },
+    article: {
+        title: '文章',
+        link: `${rootUrl}/article`,
+        api: `${apiBase}/v5/Information/newsall`,
+    },
+};
+
+export default async (ctx) => {
+    const { channel = 'newsflash' } = ctx.req.param();
+
+    const { data: response } = await got(channelMap[channel].api);
+
+    const { data } = channel === 'newsflash' ? response.data : response;
+    let list = data.map((item) => ({
+        title: item.title,
+        link: `${rootUrl}/${channel === 'newsflash' ? 'flash' : 'news'}/${item.id}`,
+        description: item.content ?? item.im_abstract,
+        pubDate: parseDate(item.add_time, 'X'),
+    }));
+
+    if (channel !== 'newsflash') {
+        list = await Promise.all(
+            list.map((item) =>
+                cache.tryGet(item.link, async () => {
+                    const { data: response } = await got(item.link);
+                    const $ = load(response);
+                    item.description = $('div.news-content').html();
+                    return item;
+                })
+            )
+        );
+    }
+
+    ctx.set('data', {
+        title: `TheBlockBeats - ${channelMap[channel].title}`,
+        link: channelMap[channel].link,
+        item: list,
+    });
+};

package/lib/routes/theblockbeats/maintainer.ts

@@ -0,0 +1,3 @@
+export default {
+    '/:channel?': ['Fatpandac', 'jameshih'],
+};

package/lib/routes/theblockbeats/radar.ts

@@ -0,0 +1,19 @@
+export default {
+    'theblockbeats.info': {
+        _name: '律动',
+        rszhaopin: [
+            {
+                title: '快讯',
+                docs: 'https://docs.rsshub.app/routes/finance#lv-dong',
+                source: ['/'],
+                target: '/theblockbeats/newsflash',
+            },
+            {
+                title: '文章',
+                docs: 'https://docs.rsshub.app/routes/finance#lv-dong',
+                source: ['/'],
+                target: '/theblockbeats/article',
+            },
+        ],
+    },
+};

package/lib/routes/theblockbeats/router.ts

@@ -0,0 +1,3 @@
+export default (router) => {
+    router.get('/:channel?', './index');
+};

package/lib/routes/thecatcity/index.ts

@@ -0,0 +1,49 @@
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import { termsMap } from './terms-map';
+
+const baseUrl = 'https://thecatcity.com';
+
+export default async (ctx) => {
+    const term = ctx.req.param('term');
+    const { data } = await got(`${baseUrl}/node_api/v1/articles/posts`, {
+        searchParams: {
+            pageId: 977_080_509_047_743,
+            term,
+        },
+    });
+
+    const list = data.data.posts.map((post) => ({
+        title: post.title,
+        description: post.description,
+        link: `${baseUrl}${post.url}`,
+        pubDate: parseDate(post.post_date),
+        guid: post.guid,
+        api: `${baseUrl}/node_api/v1/articles/${post.id}`,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.guid, async () => {
+                const { data } = await got(item.api, {
+                    searchParams: {
+                        pageId: 977_080_509_047_743,
+                    },
+                });
+                item.description = data.data.post_content;
+                item.category = [...new Set([...data.data.tags.map((t) => t.name), ...data.data.categories.map((c) => c.name)])];
+                item.author = data.data.author.display_name;
+                return item;
+            })
+        )
+    );
+
+    ctx.set('data', {
+        title: termsMap[term] ? termsMap[term].title : termsMap[''].title,
+        description: '提供貓咪日常照顧、新手準備、貓用品、貓咪醫療、貓飲食與行為等相關知識，以及療癒貓影片、貓趣聞、貓小物流行資訊，不論你是貓奴、還是貓控，一切所需都在貓奴日常找到',
+        link: baseUrl,
+        image: 'https://assets.presslogic.com/presslogic-hk-tc/static/favicon.ico',
+        item: items,
+    });
+};

package/lib/routes/thecatcity/maintainer.ts

@@ -0,0 +1,3 @@
+export default {
+    '/:term?': ['TonyRL'],
+};

package/lib/routes/thecatcity/radar.ts

@@ -0,0 +1,13 @@
+import { termsMap } from './terms-map';
+
+export default {
+    'thecatcity.com': {
+        _name: '貓奴日常',
+        '.': Object.entries(termsMap).map(([key, value]) => ({
+            title: value.title,
+            docs: 'https://docs.rsshub.app/routes/new-media#mao-nu-ri-chang',
+            source: [...new Set([value.slug, '/'])],
+            target: `/thecatcity${key ? `/${key}` : ''}`,
+        })),
+    },
+};

package/lib/routes/thecatcity/router.ts

@@ -0,0 +1,3 @@
+export default (router) => {
+    router.get('/:term?', './index');
+};

package/lib/routes/thecatcity/terms-map.ts

@@ -0,0 +1,28 @@
+const termsMap = {
+    '': {
+        title: 'CatCity 貓奴日常 | 貓咪日常照顧、新手準備、貓用品、貓咪醫療',
+        slug: '/',
+    },
+    1: {
+        title: '貓物分享｜流行小物、貓咪用品',
+        slug: '/category/cute-item',
+    },
+    2: {
+        title: '貓咪新聞｜貓界人氣熱話、貓電影',
+        slug: '/category/funny-news',
+    },
+    3: {
+        title: '養貓大全｜貓咪飲食與醫療、行為心理、貓測驗與冷知識',
+        slug: '/category/knowledge',
+    },
+    4: {
+        title: '貓奴景點｜貓咪咖啡廳與餐廳、貓奴旅行景點推薦',
+        slug: '/category/hot-spot',
+    },
+    5: {
+        title: '新手養貓教學｜養貓準備與花費、日常照顧',
+        slug: '/category/raise-cats',
+    },
+};
+
+export { termsMap };

package/lib/routes/thecover/channel.ts

@@ -0,0 +1,67 @@
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+const rootUrl = 'https://www.thecover.cn';
+
+const nodes = {
+    3892: '天下',
+    3560: '四川',
+    3909: '辟谣',
+    3686: '国际',
+    11: '云招考',
+    3902: '30秒',
+    3889: '拍客',
+    3689: '体育',
+    1: '国内',
+    4002: '帮扶铁军',
+    12: '文娱',
+    46: '宽窄',
+    4: '商业',
+    21: '千面',
+    17: '封面号',
+};
+
+export default async (ctx) => {
+    const id = ctx.req.param('id') ?? '3892';
+    const targetUrl = rootUrl.concat(`/channel_${id}`);
+    const resp = await got({
+        method: 'get',
+        url: targetUrl,
+    });
+    const $ = load(resp.data);
+    const list = $('a.link-to-article')
+        .filter(function () {
+            return $(this).attr('href').startsWith('/');
+        })
+        .map((_, item) => ({
+            link: rootUrl.concat($(item).attr('href')),
+        }))
+        .get();
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = load(detailResponse.data);
+                item.title = content('h1', '.main-article').text();
+                item.description = content('section.article-content').html();
+                const info = content('span', '.props-of-title');
+                item.author = info.eq(0).text();
+                item.pubDate = timezone(parseDate(info.eq(1).text(), 'YYYY-MM-DD HH:mm'), +8);
+                return item;
+            })
+        )
+    );
+
+    ctx.set('data', {
+        title: `${nodes[id]}-封面新闻`,
+        link: targetUrl,
+        description: `封面新闻作为华西都市报深度融合转型和打造新型主流媒体的载体，牢固确立移动优先战略，创新移动新闻产品，打造移动传播矩阵，封面新闻的传播力、引导力、影响力和公信力不断得到各方肯定。封面新闻突破千万的用户下载量，呈现出以四川为主阵地的全国分布态势，用户年龄构成以20-35岁为主，“亿万年轻人的生活方式”的定位初步得到体现。`,
+        language: 'zh-cn',
+        item: items,
+    });
+};

package/lib/routes/thecover/maintainer.ts

@@ -0,0 +1,3 @@
+export default {
+    '/channel/:id?': ['yuxinliu-alex'],
+};

package/lib/routes/thecover/radar.ts

@@ -0,0 +1,13 @@
+export default {
+    'thecover.cn': {
+        _name: '封面新闻',
+        '.': [
+            {
+                title: '频道',
+                docs: 'https://docs.rsshub.app/routes/new-media#the-cover',
+                source: ['/:id', '/'],
+                target: (params) => `/thecover/channel/${params.id.replace('channel_', '')}`,
+            },
+        ],
+    },
+};

package/lib/routes/thecover/router.ts

@@ -0,0 +1,3 @@
+export default (router) => {
+    router.get('/channel/:id?', './channel');
+};

package/lib/routes/thehindu/maintainer.ts

@@ -0,0 +1,3 @@
+export default {
+    '/topic/:topic': ['TonyRL'],
+};

package/lib/routes/thehindu/radar.ts

@@ -0,0 +1,13 @@
+export default {
+    'thehindu.com': {
+        _name: 'The Hindu',
+        '.': [
+            {
+                title: 'Topic',
+                docs: 'https://docs.rsshub.app/routes/traditional-media#the-hindu',
+                source: ['/topic/:topic'],
+                target: '/thehindu/topic/:topic',
+            },
+        ],
+    },
+};

package/lib/routes/thehindu/router.ts

@@ -0,0 +1,3 @@
+export default (router) => {
+    router.get('/topic/:topic', './topic');
+};

package/lib/routes/thehindu/topic.ts

@@ -0,0 +1,58 @@
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+export default async (ctx) => {
+    const baseUrl = 'https://www.thehindu.com';
+    const topic = ctx.req.param('topic');
+    const link = `${baseUrl}/topic/${topic}/`;
+    const apiLink = `${baseUrl}/topic/${topic}/fragment/showmoreTag`;
+
+    const { data: response } = await got(link);
+    const { data: apiResponse } = await got(apiLink);
+
+    const $ = load(response);
+
+    const $api = load(apiResponse);
+    const list = $('.element')
+        .toArray()
+        .map((item) => {
+            item = $api(item);
+            const a = item.find('.title a');
+            return {
+                title: a.text().trim(),
+                link: a.attr('href'),
+                author: item.find('.author-name').text(),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = load(response);
+
+                $('.position-relative, .articleblock-container, .article-ad, .comments-shares').remove();
+                item.description = $('.sub-title').prop('outerHTML') + $('div.article-picture').html() + $('div[itemprop="articleBody"]').html();
+                item.pubDate = parseDate($('meta[itemprop="datePublished"]').attr('content'));
+                item.updated = parseDate($('meta[itemprop="dateModified"]').attr('content'));
+                item.category = $('meta[property="article:tag"]')
+                    .toArray()
+                    .map((item) => $(item).attr('content'));
+
+                return item;
+            })
+        )
+    );
+
+    ctx.set('data', {
+        title: $('head title').text().trim(),
+        link: `${baseUrl}/topic/${topic}/`,
+        image: $('meta[property="og:image"]').attr('content'),
+        logo: $('link[rel="apple-touch-icon"]').attr('href'),
+        icon: $('link[rel="icon"]').attr('href'),
+        language: 'en',
+        item: items,
+    });
+};

package/lib/routes/theinitium/full.ts

@@ -0,0 +1,170 @@
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { config } from '@/config';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+const TOKEN = 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY=';
+
+export default async (ctx) => {
+    // model是channel/tag/etc.，而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话，那就是model是follow，type是articles。
+    const model = ctx.req.param('model') ?? 'channel';
+    const type = ctx.req.param('type') ?? 'latest';
+    const language = ctx.req.param('language') ?? 'zh-hans';
+    let listUrl;
+    let listLink;
+    switch (model) {
+        case 'author':
+            listUrl = `https://api.theinitium.com/api/v2/author/?language=${language}&slug=${type}`;
+            listLink = `https://theinitium.com/author/${type}/`;
+            break;
+        case 'follow':
+            listUrl = `https://api.theinitium.com/api/v2/user/follows/${type}/?language=${language}`;
+            listLink = `https://theinitium.com/follow/`;
+            break;
+        case 'channel':
+            listUrl = `https://api.theinitium.com/api/v2/channel/articles/?language=${language}&slug=${type}`;
+            listLink = `https://theinitium.com/channel/${type}/`;
+            break;
+        case 'tags':
+            listUrl = `https://api.theinitium.com/api/v2/tag/articles/?language=${language}&slug=${type}`;
+            listLink = `https://theinitium.com/tags/${type}/`;
+            break;
+    }
+
+    const key = {
+        email: config.initium.username,
+        password: config.initium.password,
+        iapReceipt: config.initium.iap_receipt,
+    };
+    const body = JSON.stringify(key);
+
+    let token;
+    const cacheIn = await cache.get('initium:token');
+    if (cacheIn) {
+        token = cacheIn;
+    } else if (config.initium.bearertoken) {
+        token = config.initium.bearertoken;
+        cache.set('initium:token', config.initium.bearertoken);
+    } else if (key.email === undefined) {
+        token = TOKEN;
+    } else {
+        const login = await got.post(`https://api.theinitium.com/api/v2/auth/login/?language=${language}`, {
+            headers: {
+                'Content-Type': 'application/json',
+                Accept: 'application/json',
+                Connection: 'keep-alive',
+                Authorization: TOKEN,
+                Origin: `https://theinitium.com/`,
+                Referer: `https://theinitium.com/`,
+                'X-Client-Name': 'Web',
+            },
+            body,
+        });
+
+        /*
+        const devices = login.data.access.devices;
+
+        for (const key in devices) {
+            const device = devices[key];
+            if (device.status === 'logged_in' && !device.logout_at && device.platform === 'web') {
+                token = 'Bearer ' + device.device_id;
+                break;
+            }
+        }
+        */
+        token = 'Bearer ' + login.data.token;
+        cache.set('initium:token', token);
+    }
+
+    const headers = {
+        Accept: '*/*',
+        Connection: 'keep-alive',
+        Authorization: token,
+        'X-IAP-Receipt': key.iapReceipt || '',
+    };
+
+    const response = await got(listUrl, {
+        headers,
+    });
+
+    const name = response.data.name || (response.data[model] && response.data[model].name) || '追踪';
+    // 从v1直升的channel和tags里面是digests，v2新增的author和follow出来都是results
+    const articles = response.data.results ?? response.data.digests;
+    // 如果model=author，那就是avatar；否则都是cover，要么就没封面
+    const image = response.data[model] && (response.data[model].cover || response.data[model].avatar);
+
+    const getFullText = (slug) =>
+        cache.tryGet(`theinitium:${slug}:${language}`, async () => {
+            let content = '';
+            const { data } = await got(`https://api.theinitium.com/api/v2/article/detail/?language=${language}&slug=${slug}`, {
+                headers,
+            });
+
+            if (data.lead.length) {
+                content += '<p>「' + data.lead + '」</p>';
+            }
+            if (data.byline.length) {
+                content += '<p>' + data.byline + '</p>';
+            }
+            if (data.content) {
+                content += data.content.replace('<figure class="advertisement"/><br/>', '').replaceAll(/(?:<br>){2}-{11}<br>.*$/g, '');
+            } else if (data.type === 'html') {
+                // 有时候编辑部会漏录入文章信息…………扶额。所以加这一个判断，如果确实是普通html文章，但又没有内容，说明是漏了，后面还要给guid手动加个标记，以便阅读器事后重抓。
+                content += '内容为空，请稍后再来';
+            } else if (data.type === 'web') {
+                // 有时候文章并非普通html文章，而是带有互动内容等，表现为type为web，并且content里没有内容。我们也尽力抓点东西下来。
+                // 或许可能还有其他未知情况，等碰到了再说吧。先这样留空也不碍事。
+                const nonhtmlcontent = await got(data.web.url);
+                const webcontent = load(nonhtmlcontent.body).html();
+                content += webcontent;
+            }
+            if (data.paywall_enabled) {
+                const google_bot_ua =
+                    'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.92 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
+                const accept_language = language + ';q=0.9';
+                const pay_part = await got(`https://theinitium.com/article/${slug}/`, {
+                    headers: {
+                        'user-agent': google_bot_ua,
+                        'accept-language': accept_language,
+                    },
+                });
+                const $ = load(pay_part.body);
+                const pay_content = $('div.paywall').html();
+                if (pay_content) {
+                    content += pay_content.replace('<meta itemprop="isAccessibleForFree" content="false">', '');
+                }
+            }
+            return content;
+        });
+
+    const items = await Promise.all(
+        articles
+            .filter((a) => a.article)
+            .slice(0, token === TOKEN && key.iapReceipt === undefined ? 25 : articles.length)
+            .map(async (item) => {
+                item.article.date = parseDate(item.article.date);
+                item.article.updated = parseDate(item.article.updated);
+                const description = await getFullText(item.article.slug);
+                return {
+                    title: item.article.headline,
+                    author: item.article.authors.length > 0 ? item.article.authors.map((x) => x.name).toString() : item.article.byline,
+                    category: item.article.channels.filter((x) => !x.homepage).map((x) => x.name),
+                    description,
+                    link: new URL(item.article.url, 'https://theinitium.com').href,
+                    pubDate: item.article.date,
+                    updated: item.article.updated,
+                    // 如果遇到编辑部漏录入情况，则给uuid做个手脚，以便阅读器到时重抓。
+                    guid: description.endsWith('内容为空，请稍后再来') ? item.article.uuid + '-I-am-empty' : item.article.uuid,
+                };
+            })
+    );
+
+    ctx.set('data', {
+        title: `端传媒 - ${name}`,
+        link: listLink,
+        icon: 'https://theinitium.com/misc/about/logo192.png',
+        item: items,
+        image,
+    });
+};

package/lib/routes/theinitium/maintainer.ts

@@ -0,0 +1,6 @@
+export default {
+    '/author/:type/:language?': ['AgFlore'],
+    '/channel/:type?/:language?': ['prnake'],
+    '/follow/articles/:language?': ['AgFlore'],
+    '/tags/:type/:language?': ['AgFlore'],
+};

package/lib/routes/theinitium/radar.ts

@@ -0,0 +1,25 @@
+export default {
+    'theinitium.com': {
+        _name: '端传媒',
+        '.': [
+            {
+                title: '专题・栏目',
+                docs: 'https://docs.rsshub.app/routes/new-media#duan-chuan-mei',
+                source: '/channel/:type',
+                target: '/theinitium/channel/:type',
+            },
+            {
+                title: '话题・标签',
+                docs: 'https://docs.rsshub.app/routes/new-media#duan-chuan-mei',
+                source: '/tags/:type',
+                target: '/theinitions/tags/:type',
+            },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/routes/new-media#duan-chuan-mei',
+                source: '/author/:type',
+                target: '/theinitium/author/:type',
+            },
+        ],
+    },
+};

package/lib/routes/theinitium/router.ts

@@ -0,0 +1,3 @@
+export default (router) => {
+    router.get('/:model?/:type?/:language?', './full');
+};

package/lib/routes/themoviedb/api-key.ts

@@ -0,0 +1,23 @@
+// https://github.com/rickylawson/freekeys/blob/master/index.js
+const KEYS = [
+    'fb7bb23f03b6994dafc674c074d01761',
+    'e55425032d3d0f371fc776f302e7c09b',
+    '8301a21598f8b45668d5711a814f01f6',
+    '8cf43ad9c085135b9479ad5cf6bbcbda',
+    'da63548086e399ffc910fbc08526df05',
+    '13e53ff644a8bd4ba37b3e1044ad24f3',
+    '269890f657dddf4635473cf4cf456576',
+    'a2f888b27315e62e471b2d587048f32e',
+    '8476a7ab80ad76f0936744df0430e67c',
+    '5622cafbfe8f8cfe358a29c53e19bba0',
+    'ae4bd1b6fce2a5648671bfc171d15ba4',
+    '257654f35e3dff105574f97fb4b97035',
+    '2f4038e83265214a0dcd6ec2eb3276f5',
+    '9e43f45f94705cc8e1d5a0400d19a7b7',
+    'af6887753365e14160254ac7f4345dd2',
+    '06f10fc8741a672af455421c239a1ffc',
+    'fb7bb23f03b6994dafc674c074d01761',
+    '09ad8ace66eec34302943272db0e8d2c',
+];
+
+export default () => KEYS[Math.floor(Math.random() * KEYS.length)];