rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/v2/thecatcity/index.js

@@ -0,0 +1,48 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { termsMap } = require('./termsMap');
+
+const baseUrl = 'https://thecatcity.com';
+
+module.exports = async (ctx) => {
+    const { term } = ctx.params;
+    const { data } = await got(`${baseUrl}/node_api/v1/articles/posts`, {
+        searchParams: {
+            pageId: 977080509047743,
+            term,
+        },
+    });
+
+    const list = data.data.posts.map((post) => ({
+        title: post.title,
+        description: post.description,
+        link: `${baseUrl}${post.url}`,
+        pubDate: parseDate(post.post_date),
+        guid: post.guid,
+        api: `${baseUrl}/node_api/v1/articles/${post.id}`,
+    }));
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.guid, async () => {
+                const { data } = await got(item.api, {
+                    searchParams: {
+                        pageId: 977080509047743,
+                    },
+                });
+                item.description = data.data.post_content;
+                item.category = [...new Set([...data.data.tags.map((t) => t.name), ...data.data.categories.map((c) => c.name)])];
+                item.author = data.data.author.display_name;
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: termsMap[term] ? termsMap[term].title : termsMap[''].title,
+        description: '提供貓咪日常照顧、新手準備、貓用品、貓咪醫療、貓飲食與行為等相關知識，以及療癒貓影片、貓趣聞、貓小物流行資訊，不論你是貓奴、還是貓控，一切所需都在貓奴日常找到',
+        link: baseUrl,
+        image: 'https://assets.presslogic.com/presslogic-hk-tc/static/favicon.ico',
+        item: items,
+    };
+};

package/lib/v2/thecatcity/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:term?': ['TonyRL'],
+};

package/lib/v2/thecatcity/radar.js

@@ -0,0 +1,13 @@
+const { termsMap } = require('./termsMap');
+
+module.exports = {
+    'thecatcity.com': {
+        _name: '貓奴日常',
+        '.': Object.entries(termsMap).map(([key, value]) => ({
+            title: value.title,
+            docs: 'https://docs.rsshub.app/new-media.html#mao-nu-ri-chang',
+            source: [...new Set([value.slug, '/'])],
+            target: `/thecatcity${key ? `/${key}` : ''}`,
+        })),
+    },
+};

package/lib/v2/thecatcity/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:term?', require('./index'));
+};

package/lib/v2/thecatcity/termsMap.js

@@ -0,0 +1,30 @@
+const termsMap = {
+    '': {
+        title: 'CatCity 貓奴日常 | 貓咪日常照顧、新手準備、貓用品、貓咪醫療',
+        slug: '/',
+    },
+    1: {
+        title: '貓物分享｜流行小物、貓咪用品',
+        slug: '/category/cute-item',
+    },
+    2: {
+        title: '貓咪新聞｜貓界人氣熱話、貓電影',
+        slug: '/category/funny-news',
+    },
+    3: {
+        title: '養貓大全｜貓咪飲食與醫療、行為心理、貓測驗與冷知識',
+        slug: '/category/knowledge',
+    },
+    4: {
+        title: '貓奴景點｜貓咪咖啡廳與餐廳、貓奴旅行景點推薦',
+        slug: '/category/hot-spot',
+    },
+    5: {
+        title: '新手養貓教學｜養貓準備與花費、日常照顧',
+        slug: '/category/raise-cats',
+    },
+};
+
+module.exports = {
+    termsMap,
+};

package/lib/v2/theinitium/full.js

@@ -0,0 +1,169 @@
+const got = require('@/utils/got');
+const config = require('@/config').value;
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const TOKEN = 'Basic YW5vbnltb3VzOkdpQ2VMRWp4bnFCY1ZwbnA2Y0xzVXZKaWV2dlJRY0FYTHY=';
+
+module.exports = async (ctx) => {
+    // model是channel/tag/etc.，而type是latest/feature/quest-academy这些一级栏目/标签/作者名的slug名。如果是追踪的话，那就是model是follow，type是articles。
+    const model = ctx.params.model ?? 'channel';
+    const type = ctx.params.type ?? 'latest';
+    const language = ctx.params.language ?? 'zh-hans';
+    let listUrl;
+    let listLink;
+    switch (model) {
+        case 'author':
+            listUrl = `https://api.theinitium.com/api/v2/author/?language=${language}&slug=${type}`;
+            listLink = `https://theinitium.com/author/${type}/`;
+            break;
+        case 'follow':
+            listUrl = `https://api.theinitium.com/api/v2/user/follows/${type}/?language=${language}`;
+            listLink = `https://theinitium.com/follow/`;
+            break;
+        case 'channel':
+            listUrl = `https://api.theinitium.com/api/v2/channel/articles/?language=${language}&slug=${type}`;
+            listLink = `https://theinitium.com/channel/${type}/`;
+            break;
+        case 'tags':
+            listUrl = `https://api.theinitium.com/api/v2/tag/articles/?language=${language}&slug=${type}`;
+            listLink = `https://theinitium.com/tags/${type}/`;
+            break;
+    }
+
+    const key = {
+        email: config.initium.username,
+        password: config.initium.password,
+        iapReceipt: config.initium.iap_receipt,
+    };
+    const body = JSON.stringify(key);
+
+    let token;
+    const cache = await ctx.cache.get('initium:token');
+    if (cache) {
+        token = cache;
+    } else if (config.initium.bearertoken) {
+        token = config.initium.bearertoken;
+        ctx.cache.set('initium:token', config.initium.bearertoken);
+    } else if (key.email === undefined) {
+        token = TOKEN;
+    } else {
+        const login = await got.post(`https://api.theinitium.com/api/v2/auth/login/?language=${language}`, {
+            headers: {
+                'Content-Type': 'application/json',
+                Accept: 'application/json',
+                Connection: 'keep-alive',
+                Authorization: TOKEN,
+                Origin: `https://theinitium.com/`,
+                Referer: `https://theinitium.com/`,
+                'X-Client-Name': 'Web',
+            },
+            body,
+        });
+
+        /*
+        const devices = login.data.access.devices;
+
+        for (const key in devices) {
+            const device = devices[key];
+            if (device.status === 'logged_in' && !device.logout_at && device.platform === 'web') {
+                token = 'Bearer ' + device.device_id;
+                break;
+            }
+        }
+        */
+        token = 'Bearer ' + login.data.token;
+        ctx.cache.set('initium:token', token);
+    }
+
+    const headers = {
+        Accept: '*/*',
+        Connection: 'keep-alive',
+        Authorization: token,
+        'X-IAP-Receipt': key.iapReceipt || '',
+    };
+
+    const response = await got(listUrl, {
+        headers,
+    });
+
+    const name = response.data.name || (response.data[model] && response.data[model].name) || '追踪';
+    // 从v1直升的channel和tags里面是digests，v2新增的author和follow出来都是results
+    const articles = response.data.results ? response.data.results : response.data.digests;
+    // 如果model=author，那就是avatar；否则都是cover，要么就没封面
+    const image = response.data[model] && (response.data[model].cover || response.data[model].avatar);
+
+    const getFullText = (slug) =>
+        ctx.cache.tryGet(`theinitium:${slug}:${language}`, async () => {
+            let content = '';
+            const { data } = await got(`https://api.theinitium.com/api/v2/article/detail/?language=${language}&slug=${slug}`, {
+                headers,
+            });
+
+            if (data.lead.length) {
+                content += '<p>「' + data.lead + '」</p>';
+            }
+            if (data.byline.length) {
+                content += '<p>' + data.byline + '</p>';
+            }
+            if (data.content) {
+                content += data.content.replace('<figure class="advertisement"/><br/>', '').replace(/<br><br>-----------<br>.*$/g, '');
+            } else if (data.type === 'html') {
+                // 有时候编辑部会漏录入文章信息…………扶额。所以加这一个判断，如果确实是普通html文章，但又没有内容，说明是漏了，后面还要给guid手动加个标记，以便阅读器事后重抓。
+                content += '内容为空，请稍后再来';
+            } else if (data.type === 'web') {
+                // 有时候文章并非普通html文章，而是带有互动内容等，表现为type为web，并且content里没有内容。我们也尽力抓点东西下来。
+                // 或许可能还有其他未知情况，等碰到了再说吧。先这样留空也不碍事。
+                const nonhtmlcontent = await got(data.web.url);
+                const webcontent = cheerio.load(nonhtmlcontent.body).html();
+                content += webcontent;
+            }
+            if (data.paywall_enabled) {
+                const google_bot_ua =
+                    'Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.92 Mobile Safari/537.36 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)';
+                const accept_language = language + ';q=0.9';
+                const pay_part = await got(`https://theinitium.com/article/${slug}/`, {
+                    headers: {
+                        'user-agent': google_bot_ua,
+                        'accept-language': accept_language,
+                    },
+                });
+                const $ = cheerio.load(pay_part.body);
+                const pay_content = $('div.paywall').html();
+                if (pay_content) {
+                    content += pay_content.replace('<meta itemprop="isAccessibleForFree" content="false">', '');
+                }
+            }
+            return content;
+        });
+
+    const items = await Promise.all(
+        articles
+            .filter((a) => a.article)
+            .slice(0, token === TOKEN && key.iapReceipt === undefined ? 25 : articles.length)
+            .map(async (item) => {
+                item.article.date = parseDate(item.article.date);
+                item.article.updated = parseDate(item.article.updated);
+                const description = await getFullText(item.article.slug);
+                return {
+                    title: item.article.headline,
+                    author: item.article.authors.length > 0 ? item.article.authors.map((x) => x.name).toString() : item.article.byline,
+                    category: item.article.channels.filter((x) => !x.homepage).map((x) => x.name),
+                    description,
+                    link: new URL(item.article.url, 'https://theinitium.com').href,
+                    pubDate: item.article.date,
+                    updated: item.article.updated,
+                    // 如果遇到编辑部漏录入情况，则给uuid做个手脚，以便阅读器到时重抓。
+                    guid: description.endsWith('内容为空，请稍后再来') ? item.article.uuid + '-I-am-empty' : item.article.uuid,
+                };
+            })
+    );
+
+    ctx.state.data = {
+        title: `端传媒 - ${name}`,
+        link: listLink,
+        icon: 'https://theinitium.com/misc/about/logo192.png',
+        item: items,
+        image,
+    };
+};

package/lib/v2/theinitium/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/author/:type/:language?': ['AgFlore'],
+    '/channel/:type?/:language?': ['prnake'],
+    '/follow/articles/:language?': ['AgFlore'],
+    '/tags/:type/:language?': ['AgFlore'],
+};

package/lib/v2/theinitium/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'theinitium.com': {
+        _name: '端传媒',
+        '.': [
+            {
+                title: '专题・栏目',
+                docs: 'https://docs.rsshub.app/new-media.html#duan-chuan-mei',
+                source: '/channel/:type',
+                target: '/theinitium/channel/:type',
+            },
+            {
+                title: '话题・标签',
+                docs: 'https://docs.rsshub.app/new-media.html#duan-chuan-mei',
+                source: '/tags/:type',
+                target: '/theinitions/tags/:type',
+            },
+            {
+                title: '作者',
+                docs: 'https://docs.rsshub.app/new-media.html#duan-chuan-mei',
+                source: '/author/:type',
+                target: '/theinitium/author/:type',
+            },
+        ],
+    },
+};

package/lib/v2/theinitium/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:model?/:type?/:language?', require('./full'));
+};

package/lib/v2/thepaper/channel.js

@@ -1,15 +1,20 @@
 const utils = require('./utils');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const rootUrl = 'https://m.thepaper.cn';
-
-    const link = id === '26916' ? `${rootUrl}/channel_26916` : `${rootUrl}/list_page.jsp?nodeid=${id}&isList=0&pageidx=1`;
-    const items = await utils.ProcessFeed(id === '26916' ? '.list_video_item' : '.list_item_infor', link, ctx);
+    const channel_url = `https://m.thepaper.cn/channel/${id}`;
+    const response = await got(channel_url);
+    const data = JSON.parse(cheerio.load(response.data)('#__NEXT_DATA__').html());
+    const list = data.props.pageProps.data.list;
 
+    const items = await Promise.all(list.map((item) => utils.ProcessItem(item, ctx)));
     ctx.state.data = {
-        title: `澎湃新闻频道 - ${id}`,
-        link: `${rootUrl}/channel_${id}`,
+        title: `澎湃新闻频道 - ${utils.ChannelIdToName(id, data)}`,
+        link: channel_url,
         item: items,
+        itunes_author: '澎湃新闻',
+        image: utils.ExtractLogo(response),
     };
 };

package/lib/v2/thepaper/featured.js

@@ -1,14 +1,18 @@
 const utils = require('./utils');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
 
 module.exports = async (ctx) => {
-    const rootUrl = 'https://m.thepaper.cn';
-
-    const link = `${rootUrl}/list_page.jsp?&nodeid=25949&isList=1&pageidx=1`;
-    const items = await utils.ProcessFeed('.list_item_infor', link, ctx);
+    const response = await got('https://m.thepaper.cn');
+    const data = JSON.parse(cheerio.load(response.data)('#__NEXT_DATA__').html());
+    const list = data.props.pageProps.data.list;
 
+    const items = await Promise.all(list.map((item) => utils.ProcessItem(item, ctx)));
     ctx.state.data = {
         title: '澎湃新闻 - 首页头条',
-        link: `${rootUrl}/channel_25949`,
+        link: 'https://m.thepaper.cn',
         item: items,
+        itunes_author: '澎湃新闻',
+        image: utils.ExtractLogo(response),
     };
 };

package/lib/v2/thepaper/list.js

@@ -1,15 +1,21 @@
 const utils = require('./utils');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const rootUrl = 'https://m.thepaper.cn';
-
-    const link = `${rootUrl}/list_page.jsp?nodeid=${id}&isList=1&pageidx=1`;
-    const items = await utils.ProcessFeed('.list_item_infor', link, ctx);
+    const list_url = `https://m.thepaper.cn/list/${id}`;
+    const response = await got(list_url);
+    const data = JSON.parse(cheerio.load(response.data)('#__NEXT_DATA__').html());
+    const pagePropsData = data.props.pageProps.data;
+    const list = pagePropsData.list;
 
+    const items = await Promise.all(list.map((item) => utils.ProcessItem(item, ctx)));
     ctx.state.data = {
-        title: `澎湃新闻列表 - ${id}`,
-        link: `${rootUrl}/list_${id}`,
+        title: `澎湃新闻栏目 - ${utils.ListIdToName(id, data)}`,
+        link: list_url,
         item: items,
+        itunes_author: '澎湃新闻',
+        image: pagePropsData.nodeInfo?.pic ?? utils.ExtractLogo(response),
     };
 };

package/lib/v2/thepaper/maintainer.js

@@ -1,8 +1,8 @@
 module.exports = {
     '/839studio/:id': ['umm233'],
     '/839studio': ['umm233'],
-    '/channel/:id': ['xyqfer', 'nczitzk'],
-    '/featured': ['HenryQW', 'nczitzk'],
+    '/channel/:id': ['xyqfer', 'nczitzk', 'bigfei'],
+    '/featured': ['HenryQW', 'nczitzk', 'bigfei'],
     '/factpaper/:status?': ['nczitzk'],
-    '/list/:id': ['nczitzk'],
+    '/list/:id': ['nczitzk', 'bigfei'],
 };

package/lib/v2/thepaper/radar.js

@@ -15,10 +15,10 @@ module.exports = {
                 target: (params, url) => `/thepaper/channel/${new URL(url).search(/channel_(\d+)/)}`,
             },
             {
-                title: '列表',
+                title: '栏目',
                 docs: 'https://docs.rsshub.app/traditional-media.html#peng-pai-xin-wen-lie-biao',
                 source: ['/'],
-                target: (params, url) => `/thepaper/channel/${new URL(url).search(/list_(\d+)/)}`,
+                target: (params, url) => `/thepaper/list/${new URL(url).search(/list_(\d+)/)}`,
             },
             {
                 title: '澎湃美数组作品集',

package/lib/v2/thepaper/templates/image_detail.art

@@ -0,0 +1,6 @@
+{{ each images }}
+<figure class="wp-block-image size-full">
+    <img width="{{ $value.width }}" src="{{ $value.src }}">
+</figure>
+<p>{{ $value.description }}</p>
+{{ /each }}

package/lib/v2/thepaper/templates/video_detail.art

@@ -0,0 +1,13 @@
+<video
+    src="{{ videos.url }}"
+    controls
+    playsinline="true"
+    webkit-playsinline="true"
+    x5-playsinline="true"
+    x5-video-player-type="h5"
+    x5-video-orientation="landscape|portrait"
+    x5-video-player-fullscreen="true"
+    x-webkit-airplay="allow"
+    preload="metadata"
+    poster="{{ videos.coverUrl}}"
+></video>

package/lib/v2/thepaper/utils.js

@@ -1,55 +1,92 @@
 const cheerio = require('cheerio');
-const date = require('@/utils/date');
 const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
 const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const timezone = require('@/utils/timezone');
+const path = require('path');
 
-module.exports = {
-    ProcessFeed: async (query, link, ctx) => {
-        const res = await got(link);
-
-        const $ = cheerio.load(res.data);
-        const list = $(query).slice(0, 10).get();
-
-        return Promise.all(
-            list.map((item) => {
-                const $ = cheerio.load(item);
-                const itemUrl = `https://m.thepaper.cn/${$(item).find('a').eq(0).attr('href')}`;
-                return ctx.cache.tryGet(itemUrl, async () => {
-                    const res = await got(itemUrl);
-                    const content = cheerio.load(res.data);
+const defaultRssItem = (item) => ({
+    title: item.name,
+    link: item.link,
+    description: item.name,
+    pubDate: timezone(parseDate(item.pubTimeLong), +8),
+    media: {
+        content: {
+            url: item.pic,
+        },
+    },
+});
 
-                    let description, pubDate;
+module.exports = {
+    ProcessItem: async (item, ctx) => {
+        if (item.link) {
+            // external link
+            return defaultRssItem(item);
+        } else {
+            const itemUrl = `https://m.thepaper.cn/detail/${item.contId}`;
+            return await ctx.cache.tryGet(itemUrl, async () => {
+                const res = await got(itemUrl);
+                const data = JSON.parse(cheerio.load(res.data)('#__NEXT_DATA__').html());
+                const detailData = data.props.pageProps.detailData;
+                const contentDetail = detailData.contentDetail || detailData.liveDetail || detailData.specialDetail?.specialInfo;
+                if (contentDetail) {
+                    let description = contentDetail.content || contentDetail.summary || contentDetail.desc || '';
 
-                    if (content('div.news_video_msg').length > 0) {
-                        description = content('#vdetail_sum').html();
-                        pubDate = timezone(
-                            parseDate(
-                                content('div.news_video_msg')
-                                    .html()
-                                    .replace(/ /gi, '')
-                                    .split('<br>')[0]
-                            ),
-                            +8
-                        );
-                    } else if (content('#slider_wrapper_ul').length > 0) {
-                        description = '';
-                        pubDate = new Date(date($(item).find('div.list_item_extra span').eq(1).text())).toUTCString();
-                    } else {
-                        description = content('div.newsdetail_content').html();
-                        pubDate = timezone(parseDate(content('div.date').text().trim().split('来源：')[0].trim()), +8);
+                    if (contentDetail.videos) {
+                        description =
+                            art(path.join(__dirname, 'templates/video_detail.art'), {
+                                // see https://nanmu.me/zh-cn/posts/2020/strange-html-video-tag-behavior-in-wechat/
+                                // for video tag details
+                                videos: contentDetail.videos,
+                            }) + description;
+                    }
+                    if (contentDetail.images) {
+                        description =
+                            art(path.join(__dirname, 'templates/image_detail.art'), {
+                                images: contentDetail.images,
+                            }) + description;
                     }
 
-                    const single = {
-                        title: content('title').text(),
+                    const rss_item = {
+                        title: contentDetail.name || contentDetail.shareName,
                         link: itemUrl,
                         description,
-                        pubDate,
-                        author: content('div.author').text(),
+                        category: contentDetail.tagList?.map((t) => t.tag) ?? [],
+                        pubDate: timezone(parseDate(item.pubTimeLong || contentDetail.pubTime || contentDetail.publishTime), +8),
+                        author: contentDetail.author || '',
+                        media: {
+                            content: {
+                                url: item.pic || contentDetail.videos?.coverUrl || contentDetail.bigPic,
+                            },
+                            thumbnails: {
+                                url: item.sharePic || contentDetail.sharePic,
+                            },
+                        },
                     };
-                    return single;
-                });
-            })
-        );
+                    if (contentDetail.voiceInfo?.isHaveVoice) {
+                        rss_item.enclosure_type = 'audio/mpeg';
+                        rss_item.enclosure_url = contentDetail.voiceInfo.voiceSrc;
+                        rss_item.itunes_item_image = item.pic || contentDetail.videos?.coverUrl;
+                    }
+                    return rss_item;
+                } else {
+                    return defaultRssItem(item);
+                }
+            });
+        }
+    },
+    ChannelIdToName: (nodeId, next_data) => next_data.props.appProps.menu.channelList.find((c) => c.nodeId.toString() === nodeId.toString()).name,
+    ListIdToName: (listId, next_data) => {
+        const channelList = next_data.props.appProps.menu.channelList;
+        for (const c of channelList) {
+            if (c.childNodeList && c.childNodeList.length > 0) {
+                for (const l of c.childNodeList) {
+                    if (l.nodeId.toString() === listId.toString()) {
+                        return l.name;
+                    }
+                }
+            }
+        }
     },
+    ExtractLogo: (response) => 'https://m.thepaper.cn' + cheerio.load(response.data)('img.imageCover').attr('src'),
 };