rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/v2/nytimes/daily_briefing_chinese.js

@@ -0,0 +1,67 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.nytimes.com';
+    const currentUrl = `${rootUrl}/zh-hans/series/daily-briefing-chinese`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const listData = JSON.parse(response.data.match(/"initialState":(.*),"config"/)[1]);
+
+    let items = [];
+    Object.keys(listData).forEach((key) => {
+        if (/^Article:/.test(key) && listData[key].url) {
+            const item = listData[key];
+            items.push({
+                link: item.url,
+                pubDate: parseDate(item.firstPublished),
+            });
+        }
+    });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                // to remove ads
+                content('.StoryBodyCompanionColumn').last().find('p').last().remove();
+
+                const images = detailResponse.data.match(/"url":"[^{}]+","name":"articleLarge"/g).map((e) => JSON.parse(`{${e}}`).url);
+
+                let i = 0;
+                content('figure').each(function () {
+                    content(this).html(
+                        art(path.join(__dirname, 'templates/image.art'), {
+                            url: images[i++],
+                        })
+                    );
+                });
+
+                item.title = content('meta[property="og:title"]').attr('content');
+                item.author = content('meta[name="byl"]').attr('content').replace(/By /, '');
+                item.description = content('section[name="articleBody"]').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '新闻简报 - The New York Times',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/nytimes/index.js

@@ -0,0 +1,110 @@
+const got = require('@/utils/got');
+const parser = require('@/utils/rss-parser');
+const utils = require('./utils');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    let { lang = '' } = ctx.params;
+    lang = lang.toLowerCase();
+
+    let title = '纽约时报中文网';
+    let rssUrl = 'https://cn.nytimes.com/rss/';
+
+    if (lang === 'dual') {
+        title += ' - 中英对照版';
+    } else if (lang === 'en') {
+        title += ' - 英文原版';
+    } else if (lang === 'traditionalchinese') {
+        title = '紐約時報中文網';
+        rssUrl = new URL('zh-hant', rssUrl).href;
+    } else if (lang === 'dual-traditionalchinese') {
+        title = '紐約時報中文網 - 中英對照版';
+        rssUrl = new URL('zh-hant', rssUrl).href;
+        lang = 'dual';
+    }
+
+    const browser = await require('@/utils/puppeteer')();
+    const feed = await parser.parseURL(rssUrl);
+    const items = await Promise.all(
+        feed.items.splice(0, 10).map(async (item) => {
+            let link = item.link;
+
+            let response,
+                hasEnVersion = false,
+                dual = false;
+
+            if (lang === 'dual') {
+                link = link.replace('/?utm_source=RSS', '') + '/dual';
+
+                try {
+                    response = await ctx.cache.tryGet(`nyt: ${link}`, async () => {
+                        const response = await got(link);
+
+                        return response.data;
+                    });
+
+                    dual = true;
+                } catch (error) {
+                    response = await ctx.cache.tryGet(`nyt: ${item.link}`, async () => {
+                        const response = await got(item.link);
+
+                        return response.data;
+                    });
+                }
+            } else {
+                response = await ctx.cache.tryGet(`nyt: ${item.link}`, async () => {
+                    const response = await got(item.link);
+
+                    return response.data;
+                });
+
+                if (lang === 'en') {
+                    const $ = cheerio.load(response);
+                    if ($('.dual-btn').length > 0) {
+                        hasEnVersion = true;
+                        link = $('.dual-btn a').last().attr().href;
+
+                        response = await utils.PuppeterGetter(ctx, browser, link);
+                    }
+                }
+            }
+
+            const single = {
+                title: item.title,
+                pubDate: item.pubDate,
+                link,
+                author: item['dc:creator'],
+            };
+
+            const result = utils.ProcessFeed(response, hasEnVersion);
+
+            // Match 感谢|謝.*?cn.letters@nytimes.com。
+            const ending = /感(&#x8C22|&#x8B1D);.*?cn\.letters@nytimes\.com。/g;
+
+            const matching = '<div class="article-paragraph">';
+            const formatted = '<br>' + matching;
+
+            single.description = result.description.replace(ending, '').split(matching).join(formatted);
+
+            if (hasEnVersion) {
+                single.title = result.title;
+                single.author = result.author;
+            }
+
+            if (dual) {
+                single.title = `「中英」${single.title}`;
+            }
+
+            return single;
+        })
+    );
+
+    browser.close();
+
+    ctx.state.data = {
+        title,
+        link: 'https://cn.nytimes.com',
+        description: title,
+        item: items,
+    };
+};

package/lib/v2/nytimes/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/author/:byline': ['kevinschaul'],
+    '/book/:category?': ['melvinto'],
+    '/daily_briefing_chinese': ['yueyericardo', 'nczitzk'],
+    '/:lang?': ['HenryQW'],
+};

package/lib/v2/nytimes/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'nytimes.com': {
+        _name: '纽约时报',
+        '.': [
+            {
+                title: '新闻简报',
+                docs: 'https://docs.rsshub.app/traditional-media.html#niu-yue-shi-bao',
+                source: '/zh-hans/series/daily-briefing-chinese',
+                target: '/nytimes/daily_briefing_chinese',
+            },
+            {
+                title: '畅销书排行榜',
+                docs: 'https://docs.rsshub.app/traditional-media.html#niu-yue-shi-bao',
+                source: ['/books/best-sellers/:category', '/books/best-sellers/'],
+                target: '/nytimes/book/:category',
+            },
+            {
+                title: '作者新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#niu-yue-shi-bao',
+                source: '/by/:byline',
+                target: '/nytimes/author/:byline',
+            },
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#niu-yue-shi-bao',
+                source: '/',
+                target: '/nytimes',
+            },
+        ],
+    },
+};

package/lib/v2/nytimes/router.js

@@ -0,0 +1,6 @@
+module.exports = (router) => {
+    router.get('/author/:byline', require('./author.js'));
+    router.get('/book/:category?', require('./book.js'));
+    router.get('/daily_briefing_chinese', require('./daily_briefing_chinese'));
+    router.get('/:lang?', require('./index'));
+};

package/lib/v2/nytimes/templates/image.art

@@ -0,0 +1,3 @@
+<figure>
+<img src="{{ url }}">
+</figure>

package/lib/v2/nytimes/utils.js

@@ -0,0 +1,108 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const ProcessImage = ($, e) => {
+    const photo = $(e).find('figure').find('picture').find('img');
+
+    let cover = `<figure><img src='${photo.attr('src')}'><br><figcaption>`;
+
+    const caption = $(e).find('figcaption');
+
+    cover += `${$(caption[0]).text().trim()}</figcaption></figure>`;
+
+    return cover;
+};
+
+const PuppeterGetter = async (ctx, browser, link) => {
+    const result = await ctx.cache.tryGet(`nyt: ${link}`, async () => {
+        const page = await browser.newPage();
+        await page.setRequestInterception(true);
+        page.on('request', (request) => {
+            request.resourceType() === 'document' ? request.continue() : request.abort();
+        });
+        await page.goto(link, {
+            waitUntil: 'domcontentloaded',
+        });
+        const response = await page.evaluate(() => document.querySelector('body').innerHTML);
+        return response;
+    });
+    return result;
+};
+
+const ProcessFeed = (data, hasEnVersion = false) => {
+    const $ = cheerio.load(data);
+
+    let content;
+    const result = {};
+
+    // 处理 www.nytimes.com
+    if (hasEnVersion) {
+        content = $('section[name="articleBody"]');
+        result.title = `「英」${$('h1').text().trim()}`;
+        result.author = $('article#story span[itemprop="name"]').text();
+
+        // 处理封面图片
+        $('article#story > header')
+            .find('div[data-testid="photoviewer-wrapper"]')
+            .each((i, e) => {
+                const cover = ProcessImage($, e);
+
+                $(cover).insertBefore(content[0].firstChild);
+            });
+
+        // 处理图片
+        content.find('div[data-testid="photoviewer-wrapper"]').each((i, e) => {
+            const cover = ProcessImage($, e);
+            $(cover).insertBefore(e);
+            $(e).remove();
+        });
+
+        // remove ad
+        content.find('#CNB').each((i, e) => {
+            $(e).next().remove();
+        });
+
+        content.find('div[id]').each((i, e) => {
+            $(e).remove();
+        });
+        // remove useless DOMs
+        content.find('aside').each((i, e) => {
+            $(e).remove();
+        });
+    }
+    // 处理 cn.nytimes.com
+    else {
+        content = $('section.article-body');
+
+        // remove useless DOMs
+        content.find('div.big_ad, div.article-body-aside').each((i, e) => {
+            $(e).remove();
+        });
+
+        if ($('figure.article-span-photo').length > 0) {
+            // there is a cover photo
+            const cover = $('figure.article-span-photo');
+            $(cover).insertBefore(content[0].firstChild);
+        }
+
+        if ($('footer.author-info').length > 0) {
+            // credit to original author and translators
+            const footer = $('footer.author-info');
+            $(footer).insertAfter(content[0].lastChild);
+        }
+    }
+
+    const time = $('time').attr('datetime');
+    if (time) {
+        result.pubDate = parseDate(time);
+    }
+
+    result.description = content.html();
+
+    return result;
+};
+
+module.exports = {
+    ProcessFeed,
+    PuppeterGetter,
+};

package/lib/v2/oceanengine/arithmeticIndex.js

@@ -0,0 +1,145 @@
+const dayjs = require('dayjs');
+const { art } = require('@/utils/render');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const path = require('path');
+const config = require('@/config').value;
+
+// Parameters
+const CACHE_MAX_AGE = config.cache.contentExpire;
+const DEFAULT_FETCH_DURATION_MONTH = 6; // fetch data 6 month from now
+
+// Fetch Data
+const getMultiKeywordHotTrend = async (page, keyword, start_date, end_date, app_name = 'aweme') => {
+    const e = {
+        url: 'https://trendinsight.oceanengine.com/api/open/index/get_multi_keyword_hot_trend',
+        method: 'POST',
+        data: JSON.stringify({ keyword_list: [keyword], start_date, end_date, app_name }),
+    };
+
+    const res = await page.evaluate((e) => {
+        function queryData() {
+            const p = new Promise((resolve) => {
+                const req = new XMLHttpRequest();
+                req.open(e.method, e.url, true);
+                req.setRequestHeader('accept', 'application/json, text/plain, */*');
+                req.setRequestHeader('content-type', 'application/json;charset=UTF-8');
+                req.setRequestHeader('tea-uid', '7054893410171930123');
+                req.onreadystatechange = function () {
+                    if (req.readyState === 4 && req.status === 200) {
+                        resolve(req.responseText);
+                    } else {
+                        return;
+                    }
+                };
+                req.send(e.data);
+            });
+            return p;
+        }
+        return Promise.all([queryData()]).then((result) => result);
+    }, e);
+    return res[0];
+};
+
+// Decrypt Data
+const { createDecipheriv } = require('node:crypto');
+const key = 'anN2bXA2NjYsamlh';
+const iv = 'amlheW91LHFpYW53';
+const algorithm = 'aes-128-cfb';
+const decrypt = (rawData) => {
+    const encrypted = Buffer.from(rawData, 'base64').toString('hex');
+    const decipher = createDecipheriv(algorithm, key, iv);
+    const decrypted = decipher.update(encrypted, 'hex', 'utf8') + decipher.final('utf8');
+    return JSON.parse(decrypted);
+};
+
+// Generate Feed
+const searchLinkUrls = (keyword) => [
+    `https://tophub.today/search?e=tophub&q=${keyword}`,
+    `https://www.baidu.com/s?wd=${keyword}`,
+    `https://www.google.com/search?q=${keyword}`,
+    `https://m.zhihu.com/search?type=content&q=${keyword}`,
+    `https://s.weibo.com/weibo/${keyword}`,
+    `https://www.douyin.com/search/${keyword}`,
+    `https://so.toutiao.com/search?keyword=${keyword}`,
+];
+
+const searchLinkNames = ['今日热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条'];
+
+const createContent = (keyword, queryList, queryListText) =>
+    art(path.join(__dirname, 'templates', 'content.art'), {
+        keyword,
+        queryListText,
+        queries: queryList.map((query) => ({
+            links: searchLinkUrls(encodeURIComponent(query)).map((url, index) => `<a href="${url}" rel="noopener noreferrer" target="_blank">${searchLinkNames[index]}</a>`),
+            key: query,
+        })),
+    });
+
+module.exports = async (ctx) => {
+    const now = dayjs();
+    const start_date = now.subtract(DEFAULT_FETCH_DURATION_MONTH, 'month').format('YYYYMMDD');
+    const end_date = now.format('YYYYMMDD');
+    const keyword = ctx.params.keyword;
+    if (!keyword) {
+        throw Error('Invalid keyword');
+    }
+    if (ctx.params.channel && !['douyin', 'toutiao'].includes(ctx.params.channel)) {
+        throw Error('Invalid channel。 Only support `douyin` or `toutiao`');
+    }
+
+    const channel = ctx.params.channel === 'toutiao' ? 'toutiao' : 'aweme'; // default channel is `douyin`
+    const channelName = ctx.params.channel === 'toutiao' ? '头条' : '抖音';
+
+    const link = `https://trendinsight.oceanengine.com/arithmetic-index/analysis?keyword=${keyword}&appName=${channel}`;
+
+    const item = await ctx.cache.tryGet(
+        link,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+            });
+            await page.goto('https://trendinsight.oceanengine.com/arithmetic-index');
+            const res = await getMultiKeywordHotTrend(page, keyword, start_date, end_date, channel);
+            browser.close();
+
+            const rawData = JSON.parse(res).data;
+            const data = decrypt(rawData).hot_list[0];
+
+            const searchAverage = parseInt(data.average.search_average);
+
+            return data.search_top_point_list.map(({ date, query_list }) => {
+                const searchIndex = parseInt(data.search_hot_list.find((listItem) => listItem.datetime === date).index);
+                const relativePercent = Math.round(((searchIndex - searchAverage) / searchAverage) * 100);
+                const icon = relativePercent > 0 ? '📈' : '📉';
+
+                const uniqueQueryList = query_list.filter((query) => query !== keyword);
+                const uniqueQueryListStr = uniqueQueryList.join(', ');
+
+                const content = createContent(keyword, [keyword, ...uniqueQueryList], uniqueQueryListStr);
+
+                return {
+                    title: `${icon} ${keyword} ${relativePercent}% | ${uniqueQueryListStr}`,
+                    author: `巨量算数 - ${channelName}算数指数`,
+                    description: content,
+                    link,
+                    pubDate: timezone(parseDate(date), +8),
+                    guid: `巨量算数 - ${channelName}算数指数 | ${keyword} - ${date}`,
+                };
+            });
+        },
+        CACHE_MAX_AGE,
+        false
+    );
+
+    ctx.state.data = {
+        title: `${keyword} - ${channelName}指数波峰`,
+        link,
+        description: `巨量算数 - ${channelName}算数指数 | 关键词: ${keyword}`,
+        language: 'zh-cn',
+        item,
+    };
+};

package/lib/v2/oceanengine/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/index/:keyword/:channel?': ['Jkker'],
+};

package/lib/v2/oceanengine/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'oceanengine.com': {
+        _name: '巨量算数',
+        trendinsight: [
+            {
+                title: '抖音/头条指数波峰',
+                docs: 'https://docs.rsshub.app/other.html#ju-liang-suan-shu-suan-shu-zhi-shu',
+                source: ['/arithmetic-index/analysis'],
+                target: (params, url) => `/oceanengine/index/${new URL(url).searchParams.get('keyword')}`,
+            },
+        ],
+    },
+};

package/lib/v2/oceanengine/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/index/:keyword/:channel?', require('./arithmeticIndex'));
+};

package/lib/v2/oceanengine/templates/content.art

@@ -0,0 +1,14 @@
+<article>
+  <h2>关键词: </h2>
+  {{queryListText}}
+  &nbsp&nbsp&nbsp&nbsp&nbsp
+  <br>
+  {{each queries q}}
+  <h2>{{q.key}}</h2>
+  <p>
+    {{each q.links l}}
+    {{@l}}&nbsp&nbsp
+    {{/each}}
+  </p>
+  {{/each}}
+</article>

package/lib/v2/openwrt/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/releases/:model?': ['DIYgod'],
+};

package/lib/v2/openwrt/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'openwrt.org': {
+        _name: 'Openwrt',
+        '.': [
+            {
+                title: 'Model Releases',
+                docs: 'https://docs.rsshub.app/program-update.html#openwrt',
+                source: '/toh/:band/:model',
+                target: '/openwrt/releases/:model',
+            },
+        ],
+    },
+};

package/lib/v2/openwrt/releases.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const brand = ctx.params.brand;
+    const model = ctx.params.model;
+    const url = `https://openwrt.org/toh/${brand}/${model}`;
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const table = $('#installation').next();
+
+    ctx.state.data = {
+        title: $('h1').text() + ' - OpenWrt Releases',
+        link: url,
+        description: $('.dw-content div.level1').text(),
+        language: 'en-US',
+        item: [
+            {
+                title: table.find('.supported_current_rel').text(),
+                link: url,
+                description: `Firmware OpenWrt Install: ${table.find('.firmware_openwrt_install_url').html()}<br><br>Firmware OpenWrt Upgrade: ${$('.firmware_openwrt_upgrade_url').html()}'`,
+            },
+        ],
+    };
+};

package/lib/v2/openwrt/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/releases/:brand/:model', require('./releases'));
+};

package/lib/v2/orcid/index.js

@@ -34,7 +34,7 @@ module.exports = async (ctx) => {
             link: work.url,
             description: art(path.join(__dirname, 'templates/description.art'), {
                 title: work.title.value,
-                journalTitle: work.journalTitle.value,
+                journalTitle: work.journalTitle?.value,
                 publicationDate: work.publicationDate,
                 workType: work.workType.value,
                 Str,

package/lib/v2/orcid/templates/description.art

@@ -1,3 +1,3 @@
-<h2>{{ title }}</h2><h3>{{ journalTitle }}</h3>
+<h2>{{ title }}</h2>{{ if journalTitle }}<h3>{{ journalTitle }}</h3>{{ /if }}
 <span>{{ if publicationDate.year }}{{ publicationDate.year }}{{ /if }}{{ if publicationDate.month }}-{{ publicationDate.month }}{{ /if }}{{ if publicationDate.day }}-{{ publicationDate.day }}{{ /if }} | {{ workType }}</span><br>
-<span>{{@ Str }}</span><span>Source: {{ sourceName }}</span>
+<span>{{@ Str }}</span><span>Source: {{ sourceName }}</span>

package/lib/v2/panewslab/author.js

@@ -0,0 +1,48 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const rootUrl = 'https://panewslab.com';
+    const apiUrl = `${rootUrl}/webapi/subject/articles?Rn=${ctx.query.limit ?? 25}&LId=1&tw=0&uid=${id}`;
+    const currentUrl = `${rootUrl}/zh/author/${id}.html`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let items = response.data.data.map((item) => ({
+        title: item.title,
+        author: item.author.name,
+        pubDate: parseDate(item.publishTime * 1000),
+        link: `${rootUrl}/zh/articledetails/${item.id}.html`,
+        description: `<blockquote>${item.desc}</blockquote>`,
+        category: item.tags,
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description += content('#txtinfo').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `PANews - ${items[0].author}`,
+        link: currentUrl,
+        item: items,
+    };
+};