rsshub 1.0.0-master.fdbb6fa → 1.0.0-master.fdc59d2

package/lib/v2/thepaper/list.js

@@ -1,15 +1,21 @@
 const utils = require('./utils');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const rootUrl = 'https://m.thepaper.cn';
-
-    const link = `${rootUrl}/list_page.jsp?nodeid=${id}&isList=1&pageidx=1`;
-    const items = await utils.ProcessFeed('.list_item_infor', link, ctx);
+    const list_url = `https://m.thepaper.cn/list/${id}`;
+    const response = await got(list_url);
+    const data = JSON.parse(cheerio.load(response.data)('#__NEXT_DATA__').html());
+    const pagePropsData = data.props.pageProps.data;
+    const list = pagePropsData.list;
 
+    const items = await Promise.all(list.map((item) => utils.ProcessItem(item, ctx)));
     ctx.state.data = {
-        title: `澎湃新闻列表 - ${id}`,
-        link: `${rootUrl}/list_${id}`,
+        title: `澎湃新闻栏目 - ${utils.ListIdToName(id, data)}`,
+        link: list_url,
         item: items,
+        itunes_author: '澎湃新闻',
+        image: pagePropsData.nodeInfo?.pic ?? utils.ExtractLogo(response),
     };
 };

package/lib/v2/thepaper/maintainer.js

@@ -1,8 +1,8 @@
 module.exports = {
     '/839studio/:id': ['umm233'],
     '/839studio': ['umm233'],
-    '/channel/:id': ['xyqfer', 'nczitzk'],
-    '/featured': ['HenryQW', 'nczitzk'],
+    '/channel/:id': ['xyqfer', 'nczitzk', 'bigfei'],
+    '/featured': ['HenryQW', 'nczitzk', 'bigfei'],
     '/factpaper/:status?': ['nczitzk'],
-    '/list/:id': ['nczitzk'],
+    '/list/:id': ['nczitzk', 'bigfei'],
 };

package/lib/v2/thepaper/radar.js

@@ -15,10 +15,10 @@ module.exports = {
                 target: (params, url) => `/thepaper/channel/${new URL(url).search(/channel_(\d+)/)}`,
             },
             {
-                title: '列表',
+                title: '栏目',
                 docs: 'https://docs.rsshub.app/traditional-media.html#peng-pai-xin-wen-lie-biao',
                 source: ['/'],
-                target: (params, url) => `/thepaper/channel/${new URL(url).search(/list_(\d+)/)}`,
+                target: (params, url) => `/thepaper/list/${new URL(url).search(/list_(\d+)/)}`,
             },
             {
                 title: '澎湃美数组作品集',

package/lib/v2/thepaper/templates/image_detail.art

@@ -0,0 +1,6 @@
+{{ each images }}
+<figure class="wp-block-image size-full">
+    <img width="{{ $value.width }}" src="{{ $value.src }}">
+</figure>
+<p>{{ $value.description }}</p>
+{{ /each }}

package/lib/v2/thepaper/templates/video_detail.art

@@ -0,0 +1,13 @@
+<video
+    src="{{ videos.url }}"
+    controls
+    playsinline="true"
+    webkit-playsinline="true"
+    x5-playsinline="true"
+    x5-video-player-type="h5"
+    x5-video-orientation="landscape|portrait"
+    x5-video-player-fullscreen="true"
+    x-webkit-airplay="allow"
+    preload="metadata"
+    poster="{{ videos.coverUrl}}"
+></video>

package/lib/v2/thepaper/utils.js

@@ -1,55 +1,88 @@
 const cheerio = require('cheerio');
-const date = require('@/utils/date');
 const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
 const got = require('@/utils/got');
+const { art } = require('@/utils/render');
+const timezone = require('@/utils/timezone');
+const path = require('path');
 
 module.exports = {
-    ProcessFeed: async (query, link, ctx) => {
-        const res = await got(link);
+    ProcessItem: async (item, ctx) => {
+        if (item.link) {
+            // external link
+            return {
+                title: item.name,
+                link: item.link,
+                description: item.name,
+                pubDate: timezone(parseDate(item.pubTimeLong), +8),
+                media: {
+                    content: {
+                        url: item.pic,
+                    },
+                },
+            };
+        } else {
+            const itemUrl = `https://m.thepaper.cn/detail/${item.contId}`;
+            return await ctx.cache.tryGet(itemUrl, async () => {
+                const res = await got(itemUrl);
+                const data = JSON.parse(cheerio.load(res.data)('#__NEXT_DATA__').html());
+                const detailData = data.props.pageProps.detailData;
 
-        const $ = cheerio.load(res.data);
-        const list = $(query).slice(0, 10).get();
+                const contentDetail = detailData.contentDetail || detailData.liveDetail;
+                let description = contentDetail.content || contentDetail.summary || '';
 
-        return Promise.all(
-            list.map((item) => {
-                const $ = cheerio.load(item);
-                const itemUrl = `https://m.thepaper.cn/${$(item).find('a').eq(0).attr('href')}`;
-                return ctx.cache.tryGet(itemUrl, async () => {
-                    const res = await got(itemUrl);
-                    const content = cheerio.load(res.data);
+                if (contentDetail.videos) {
+                    description =
+                        art(path.join(__dirname, 'templates/video_detail.art'), {
+                            // see https://nanmu.me/zh-cn/posts/2020/strange-html-video-tag-behavior-in-wechat/
+                            // for video tag details
+                            videos: contentDetail.videos,
+                        }) + description;
+                }
 
-                    let description, pubDate;
+                if (contentDetail.images) {
+                    description =
+                        art(path.join(__dirname, 'templates/image_detail.art'), {
+                            images: contentDetail.images,
+                        }) + description;
+                }
 
-                    if (content('div.news_video_msg').length > 0) {
-                        description = content('#vdetail_sum').html();
-                        pubDate = timezone(
-                            parseDate(
-                                content('div.news_video_msg')
-                                    .html()
-                                    .replace(/ /gi, '')
-                                    .split('<br>')[0]
-                            ),
-                            +8
-                        );
-                    } else if (content('#slider_wrapper_ul').length > 0) {
-                        description = '';
-                        pubDate = new Date(date($(item).find('div.list_item_extra span').eq(1).text())).toUTCString();
-                    } else {
-                        description = content('div.newsdetail_content').html();
-                        pubDate = timezone(parseDate(content('div.date').text().trim().split('来源：')[0].trim()), +8);
+                const rss_item = {
+                    title: contentDetail.name,
+                    link: itemUrl,
+                    description,
+                    category: contentDetail.tagList?.map((t) => t.tag) ?? [],
+                    pubDate: timezone(parseDate(item.pubTimeLong || contentDetail.pubTime), +8),
+                    author: contentDetail.author,
+                    media: {
+                        content: {
+                            url: item.pic || contentDetail.videos?.coverUrl,
+                        },
+                        thumbnails: {
+                            url: item.sharePic || contentDetail.sharePic,
+                        },
+                    },
+                };
+                if (contentDetail.voiceInfo?.isHaveVoice) {
+                    rss_item.enclosure_type = 'audio/mpeg';
+                    rss_item.enclosure_url = contentDetail.voiceInfo.voiceSrc;
+                    rss_item.itunes_item_image = item.pic || contentDetail.videos?.coverUrl;
+                }
+                return rss_item;
+            });
+        }
+    },
+    ChannelIdToName: (nodeId, next_data) => next_data.props.appProps.menu.channelList.find((c) => c.nodeId.toString() === nodeId.toString()).name,
+    ListIdToName: (listId, next_data) => {
+        const channelList = next_data.props.appProps.menu.channelList;
+        for (const c of channelList) {
+            if (c.childNodeList && c.childNodeList.length > 0) {
+                for (const l of c.childNodeList) {
+                    if (l.nodeId.toString() === listId.toString()) {
+                        return l.name;
                     }
-
-                    const single = {
-                        title: content('title').text(),
-                        link: itemUrl,
-                        description,
-                        pubDate,
-                        author: content('div.author').text(),
-                    };
-                    return single;
-                });
-            })
-        );
+                }
+            }
+        }
     },
+    ExtractLogo: (response) => 'https://m.thepaper.cn' + cheerio.load(response.data)('img.imageCover').attr('src'),
 };

package/lib/v2/theverge/index.js

@@ -0,0 +1,90 @@
+const got = require('@/utils/got');
+const parser = require('@/utils/rss-parser');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const feed = await parser.parseURL('https://www.theverge.com/rss/index.xml');
+
+    const items = await Promise.all(
+        feed.items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+
+                const $ = cheerio.load(response.data);
+
+                const content = $('#content');
+                const body = $('.duet--article--article-body-component-container');
+
+                // 处理封面图片
+
+                const cover = $('meta[property="og:image"]');
+
+                if (cover.length > 0) {
+                    $(`<img src=${cover[0].attribs.content}>`).insertBefore(body[0].childNodes[0]);
+                }
+
+                // 处理封面视频
+                $('div.l-col__main > div.c-video-embed, div.c-entry-hero > div.c-video-embed').each((i, e) => {
+                    const src = `https://volume.vox-cdn.com/embed/${e.attribs['data-volume-uuid']}?autoplay=false`;
+
+                    $(`<iframe src="${src}" style="border: 0; top: 0; left: 0; width: 100%; height: 100%; position: absolute;" allowfullscreen scrolling="no"></iframe>`).insertBefore(body[0].childNodes[0]);
+                });
+
+                // 处理封面视频
+                $('div.l-col__main > div.c-video-embed--media iframe').each((i, e) => {
+                    $(e).insertBefore(body[0].childNodes[0]);
+                });
+
+                // 处理文章图片
+                content.find('figure.e-image').each((i, e) => {
+                    let src, caption;
+
+                    // 处理 jpeg, png
+                    if ($(e).find('picture > source').length > 0) {
+                        src = $(e)
+                            .find('picture > img')[0]
+                            .attribs.srcset.match(/(?<=320w,).*?(?=520w)/g)[0]
+                            .trim();
+                    } else if ($(e).find('img.c-dynamic-image').length > 0) {
+                        // 处理 gif
+                        src = $(e).find('span.e-image__image')[0].attribs['data-original'];
+                    }
+
+                    // 处理 caption
+                    if ($(e).find('span.e-image__meta').length > 0) {
+                        caption = $(e).find('span.e-image__meta').text();
+                    }
+
+                    const figure = `<figure><img src=${src}>${caption ? `<br><figcaption>${caption}</figcaption>` : ''}</figure>`;
+
+                    $(figure).insertBefore(e);
+
+                    $(e).remove();
+                });
+
+                const lede = $('.duet--article--lede h2:first');
+                if (lede[0]) {
+                    lede.insertBefore(body[0].childNodes[0]);
+                }
+
+                // 移除无用 DOM
+                content.find('.duet--article--comments-join-the-conversation').remove();
+                content.find('.duet--recirculation--related-list').remove();
+                delete item.content;
+                delete item.contentSnippet;
+                delete item.isoDate;
+
+                item.description = body.html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: feed.title,
+        link: feed.link,
+        description: feed.description,
+        item: items,
+    };
+};

package/lib/v2/theverge/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['HenryQW'],
+};

package/lib/v2/theverge/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'theverge.com': {
+        _name: 'The Verge',
+        '.': [
+            {
+                title: 'The Verge',
+                docs: 'https://docs.rsshub.app/new-media.html#the-verge',
+                source: ['/rss/index.xml', '/'],
+                target: '/theverge',
+            },
+        ],
+    },
+};

package/lib/v2/theverge/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/', require('./index'));
+};

package/lib/v2/thwiki/index.js

@@ -0,0 +1,32 @@
+const got = require('@/utils/got');
+const dayjs = require('dayjs');
+
+module.exports = async (ctx) => {
+    const beforeDays = ctx.params.before ? parseInt(ctx.params.before) : 30;
+    const afterDays = ctx.params.after ? parseInt(ctx.params.after) : 30;
+    const before = dayjs().subtract(beforeDays, 'day').format('YYYY-MM-DD');
+    const after = dayjs().add(afterDays, 'day').format('YYYY-MM-DD');
+
+    const response = await got({
+        method: 'get',
+        url: `https://calendar-serverless.thwiki.cc/api/events/${before}/${after}`,
+        headers: {
+            Origin: 'https://thwiki.cc',
+        },
+    });
+    const data = response.data.results;
+
+    ctx.state.data = {
+        title: 'Touhou events calendar (THBWiki)',
+        link: `https://calendar.thwiki.cc/`,
+        description: 'A Touhou related events calendar api from THBWiki',
+        item: data.map((item) => ({
+            title: item.title,
+            author: item.title,
+            category: item.type ? item.type[0] : '活动',
+            description: `${item.desc}. 开始时间: ${item.startStr}. 结束时间: ${item.endStr}.${item.type ? ' 活动类型: ' + item.type[0] : ''}`,
+            guid: item.id,
+            link: item.url,
+        })),
+    };
+};

package/lib/v2/thwiki/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/calendar/:before?/:after?': ['aether17'],
+};

package/lib/v2/thwiki/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'thwiki.cc': {
+        _name: 'THBWiki',
+        '.': [
+            {
+                title: '日历',
+                docs: 'https://docs.rsshub.app/#thbwiki',
+                source: ['/', '/日程表'],
+                target: '/thwiki/calendar',
+            },
+        ],
+    },
+};

package/lib/v2/thwiki/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/calendar/:before?/:after?', require('./index'));
+};

package/lib/v2/tiktok/user.js

@@ -18,7 +18,7 @@ module.exports = async (ctx) => {
                 request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
             });
             await page.goto(`${baseUrl}/${user}`, {
-                waitUntil: 'domcontentloaded',
+                waitUntil: 'networkidle0',
             });
             const SIGI_STATE = await page.evaluate(() => window.SIGI_STATE);
             browser.close();

package/lib/v2/trending/allTrending.js

@@ -8,6 +8,7 @@ const config = require('@/config').value;
 const md5 = require('@/utils/md5');
 
 // Constants
+const CACHE_KEY = 'trending-all-in-one';
 const DATA_REPO_BASE_URL = 'https://raw.githubusercontent.com/huqi-pr/trending-in-one/master/raw';
 const DATE_FORMAT = 'YYYY-MM-DD';
 // TODO: support custom data repo urls
@@ -35,7 +36,6 @@ const CHANNELS = {
 };
 
 // Helper Functions
-// TODO: integrate into CHANNELS
 const processRawDataByChannel = {
     'toutiao-search': ({ word: title }) => ({
         // 源 url 存在 encoding 问题，暂时不使用
@@ -61,38 +61,63 @@ const processRawDataByChannel = {
 
 const hasKeyword = (str, keywordList) => keywordList.some((keyword) => str.includes(keyword));
 const toShanghaiTimezone = (date) => dayjs.tz(date, 'Asia/Shanghai');
+const processRawData = (channel) => (res) => res.map((item) => processRawDataByChannel[channel](item));
+const filterKeyword = (keywordList) => (res) => res.filter(({ title }) => hasKeyword(title, keywordList));
 
 // Data Fetcher
 // TODO: support channel selection
 const fetchAllData = async (keywordList = [], dateList = [], cache) => {
-    const cachedGetData = (url, dataMappingFn) =>
-        cache.tryGet(
-            url,
-            () =>
-                got(url)
-                    .json()
-                    // Normalize data to { title, url } format
-                    .then((res) => res.map((item) => dataMappingFn(item))),
-            config.cache.contentExpire,
-            false
-        );
-
-    const data = await Promise.all(
+    const cachedGetData = (url) => cache.tryGet(url, () => got(url).json(), config.cache.contentExpire, false);
+
+    let data = await Promise.all(
         dateList.map(async (dateTime) => ({
             dateTime,
             data: await Promise.all(
                 Object.keys(CHANNELS).map(async (channel) => ({
                     name: CHANNELS[channel].name,
-                    data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`, processRawDataByChannel[channel]).then((res) => res.filter(({ title }) => hasKeyword(title, keywordList))),
+                    data: await cachedGetData(`${DATA_REPO_BASE_URL}/${channel}/${dateTime.format(DATE_FORMAT)}.json`)
+                        .then(processRawData(channel))
+                        .then(filterKeyword(keywordList)),
                 }))
             ),
         }))
     );
+
     for (const i of data) {
         i.count = i.data.reduce((acc, { data }) => acc + data.length, 0);
     }
 
-    return data.filter(({ count }) => count > 0);
+    data = data.filter(({ count }) => count > 0);
+
+    if (data.length === 0) {
+        return data;
+    }
+
+    const prev = cache.get(CACHE_KEY + ':latest-items');
+
+    const latest = data[0];
+    latest.newItemCount = 0;
+    if (latest.count > 0 && prev) {
+        // Mark new items in latest
+        for (const channel in latest.data) {
+            for (const i of latest.data[channel].data) {
+                i.new = !(i.url in prev);
+                latest.newItemCount += i.new ? 1 : 0;
+            }
+        }
+        // Save latest data to cache
+        const cachedItems = latest.data.reduce((acc, { data: channel }) => {
+            for (const item of channel) {
+                acc[item.url] = true;
+            }
+            return acc;
+        }, {});
+        cache.set(CACHE_KEY + ':latest-items', cachedItems, config.cache.contentExpire);
+
+        latest.count = Object.keys(cachedItems).length;
+    }
+
+    return data;
 };
 
 // Generate Feed Items
@@ -108,11 +133,13 @@ const searchLinkUrls = (keyword) => [
 
 const searchLinkNames = ['热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条'];
 
-const createItem = ({ dateTime, data, count }, keywords, isToday) => {
+const createItem = ({ dateTime, data, count, newItemCount }, keywords, isToday) => {
     const EOD = dateTime.endOf('day');
     const pubDate = isToday ? new Date() : EOD.toDate();
+    const countStr = isToday && newItemCount ? newItemCount + '🆕' : count;
+
     return {
-        title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${count})`,
+        title: `${keywords.join(', ')} | ${dateTime.format(DATE_FORMAT)} 热点追踪 (${countStr})`,
         author: 'Trending All In One',
         pubDate,
         description: art(path.join(__dirname, 'templates/content.art'), {
@@ -147,7 +174,7 @@ module.exports = async (ctx) => {
         .map((keyword) => keyword.trim());
     const keywordStr = keywordList.join(', ');
 
-    const data = await fetchAllData(keywordList, dateList, ctx.cache);
+    const data = await fetchAllData(keywordList, dateList, ctx.cache).catch(() => []);
     const item =
         data.length > 0
             ? data.map((i, index) => createItem(i, keywordList, index === 0))

package/lib/v2/trending/templates/content.art

@@ -7,6 +7,7 @@
       {{each channel.data item}}
       <li>
         <a href="{{item.url}}" rel="noopener noreferrer" target="_blank">{{item.title}}</a>
+        {{if item.new}}🆕{{/if}}
       </li>
       {{/each}}
     </ul>

package/lib/v2/trow/portal.js

@@ -4,11 +4,21 @@ const { parseDate } = require('@/utils/parse-date');
 const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
-    const response = await got({
-        method: 'get',
+    let data;
+    const response = await got.extend({ followRedirect: false }).get({
         url: `https://trow.cc`,
     });
-    const data = response.data;
+    if (response.statusCode === 302) {
+        const response2 = await got.extend({ followRedirect: false }).get({
+            url: `https://trow.cc`,
+            headers: {
+                cookie: response.headers['set-cookie'],
+            },
+        });
+        data = response2.data;
+    } else {
+        data = response.data;
+    }
 
     const $ = cheerio.load(data);
     const list = $('#portal_content .borderwrap[style="display:show"]');

package/lib/v2/twitter/maintainer.js

@@ -1,7 +1,7 @@
 module.exports = {
     '/collection/:uid/:collectionId/:routeParams?': ['TonyRL'],
     '/followings/:id/:routeParams?': ['DIYgod'],
-    '/keyword/:keyword/:routeParams?/:limit?': ['DIYgod', 'yindaheng98', 'Rongronggg9'],
+    '/keyword/:keyword/:routeParams?': ['DIYgod', 'yindaheng98', 'Rongronggg9'],
     '/likes/:id/:routeParams?': ['xyqfer'],
     '/list/:id/:name/:routeParams?': ['xyqfer'],
     '/media/:id/:routeParams?': ['yindaheng98', 'Rongronggg9'],

package/lib/v2/twitter/router.js

@@ -1,7 +1,7 @@
 module.exports = (router) => {
     router.get('/collection/:uid/:collectionId/:routeParams?', require('./collection'));
     router.get('/followings/:id/:routeParams?', require('./followings'));
-    router.get('/keyword/:keyword/:routeParams?/:limit?', require('./keyword'));
+    router.get('/keyword/:keyword/:routeParams?', require('./keyword'));
     router.get('/likes/:id/:routeParams?', require('./likes'));
     router.get('/list/:id/:name/:routeParams?', require('./list'));
     router.get('/media/:id/:routeParams?', require('./media'));

package/lib/v2/twitter/web-api/twitter-api.js

@@ -106,6 +106,7 @@ const timelineKeywords = (keywords, params = {}) =>
         ..._params,
         ...params,
         q: keywords,
+        tweet_search_mode: 'live',
         query_source: 'typed_query',
         pc: 1,
     });

package/lib/v2/twitter/web-api/twitter-got.js

@@ -6,8 +6,8 @@ const config = require('@/config').value;
 // https://github.com/mikf/gallery-dl/blob/a53cfc845e12d9e98fefd07e43ebffaec488c18f/gallery_dl/extractor/twitter.py#L716-L726
 const headers = {
     authorization: config.twitter.authorization,
-    // every single user and guest share the same token:
-    // Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA
+    // Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw
+    // reference: https://github.com/dangeredwolf/FixTweet/blob/f3082bbb0d69798687481a605f6760b2eb7558e0/src/constants.ts#L23-L25
     'x-guest-token': undefined,
     'x-twitter-auth-type': undefined,
     'x-twitter-client-language': 'en',