rsshub 1.0.0-master.fc5a2f6 → 1.0.0-master.fc648b5

package/lib/v2/nytimes/daily_briefing_chinese.js

@@ -1,61 +1,67 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
-    const url = 'https://www.nytimes.com/zh-hans/series/daily-briefing-chinese';
+    const rootUrl = 'https://www.nytimes.com';
+    const currentUrl = `${rootUrl}/zh-hans/series/daily-briefing-chinese`;
+
     const response = await got({
         method: 'get',
-        url,
+        url: currentUrl,
+    });
+
+    const listData = JSON.parse(response.data.match(/"initialState":(.*),"config"/)[1]);
+
+    let items = [];
+    Object.keys(listData).forEach((key) => {
+        if (/^Article:/.test(key) && listData[key].url) {
+            const item = listData[key];
+            items.push({
+                link: item.url,
+                pubDate: parseDate(item.firstPublished),
+            });
+        }
     });
-    const data = response.data;
-    const $ = cheerio.load(data);
-    const post = $('.css-13mho3u .css-ye6x8s')
-        .map((index, elem) => {
-            const $item = $(elem);
-            const $link = $item.find('a');
-            const $title = $item.find('h2');
-
-            return {
-                title: $title.text(),
-                link: 'https://www.nytimes.com' + $link.attr('href'),
-            };
-        })
-        .get();
-
-    const items = await Promise.all(
-        post.map((item) =>
+
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const response = await got.get(item.link);
-                const data = response.data;
-                const arr = data.match(/"url":"https:\\u002F\\u002Fstatic(\S*)"articleLarge"/g);
-                const imageLink = arr.map((e) => JSON.parse(`{${e}}`).url);
-
-                const $ = cheerio.load(data);
-                $('.css-pncxxs.etfikam0').remove();
-                // remove 'Credit...'
-                $('.css-1ly73wi.e1tej78p0').remove();
-                $('.css-1xdhyk6.erfvjey0').each((i, e) => $(e).replaceWith(`<img src="${imageLink[i]}">`));
-
-                $('.css-1l44abu.ewdxa0s0').each((i, e) => {
-                    const figureNote = $(e)
-                        .children()
-                        .map((i, e) => $(e).text())
-                        .get();
-                    $(e).replaceWith(`<p><small>${figureNote.join(' ')}</small><p>`);
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
                 });
-                const date = $('time').attr('datetime');
 
-                item.description = $('.meteredContent.css-1r7ky0e').html();
-                item.pubDate = parseDate(date);
+                const content = cheerio.load(detailResponse.data);
+
+                // to remove ads
+                content('.StoryBodyCompanionColumn').last().find('p').last().remove();
+
+                const images = detailResponse.data.match(/"url":"[^{}]+","name":"articleLarge"/g).map((e) => JSON.parse(`{${e}}`).url);
+
+                let i = 0;
+                content('figure').each(function () {
+                    content(this).html(
+                        art(path.join(__dirname, 'templates/image.art'), {
+                            url: images[i++],
+                        })
+                    );
+                });
+
+                item.title = content('meta[property="og:title"]').attr('content');
+                item.author = content('meta[name="byl"]').attr('content').replace(/By /, '');
+                item.description = content('section[name="articleBody"]').html();
+
                 return item;
             })
         )
     );
 
     ctx.state.data = {
-        title: '纽约时报中文网|新闻简报',
-        link: url,
+        title: '新闻简报 - The New York Times',
+        link: currentUrl,
         item: items,
     };
 };

package/lib/v2/nytimes/index.js

@@ -38,7 +38,7 @@ module.exports = async (ctx) => {
 
                 try {
                     response = await ctx.cache.tryGet(`nyt: ${link}`, async () => {
-                        const response = await got.get(link);
+                        const response = await got(link);
 
                         return response.data;
                     });
@@ -46,14 +46,14 @@ module.exports = async (ctx) => {
                     dual = true;
                 } catch (error) {
                     response = await ctx.cache.tryGet(`nyt: ${item.link}`, async () => {
-                        const response = await got.get(item.link);
+                        const response = await got(item.link);
 
                         return response.data;
                     });
                 }
             } else {
                 response = await ctx.cache.tryGet(`nyt: ${item.link}`, async () => {
-                    const response = await got.get(item.link);
+                    const response = await got(item.link);
 
                     return response.data;
                 });

package/lib/v2/nytimes/maintainer.js

@@ -1,6 +1,6 @@
 module.exports = {
     '/author/:byline': ['kevinschaul'],
     '/book/:category?': ['melvinto'],
-    '/daily_briefing_chinese': ['yueyericardo'],
+    '/daily_briefing_chinese': ['yueyericardo', 'nczitzk'],
     '/:lang?': ['HenryQW'],
 };

package/lib/v2/nytimes/templates/image.art

@@ -0,0 +1,3 @@
+<figure>
+<img src="{{ url }}">
+</figure>

package/lib/v2/nytimes/utils.js

@@ -16,7 +16,13 @@ const ProcessImage = ($, e) => {
 const PuppeterGetter = async (ctx, browser, link) => {
     const result = await ctx.cache.tryGet(`nyt: ${link}`, async () => {
         const page = await browser.newPage();
-        await page.goto(link);
+        await page.setRequestInterception(true);
+        page.on('request', (request) => {
+            request.resourceType() === 'document' ? request.continue() : request.abort();
+        });
+        await page.goto(link, {
+            waitUntil: 'domcontentloaded',
+        });
         const response = await page.evaluate(() => document.querySelector('body').innerHTML);
         return response;
     });

package/lib/v2/oceanengine/arithmeticIndex.js

@@ -0,0 +1,145 @@
+const dayjs = require('dayjs');
+const { art } = require('@/utils/render');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const path = require('path');
+const config = require('@/config').value;
+
+// Parameters
+const CACHE_MAX_AGE = config.cache.contentExpire;
+const DEFAULT_FETCH_DURATION_MONTH = 6; // fetch data 6 month from now
+
+// Fetch Data
+const getMultiKeywordHotTrend = async (page, keyword, start_date, end_date, app_name = 'aweme') => {
+    const e = {
+        url: 'https://trendinsight.oceanengine.com/api/open/index/get_multi_keyword_hot_trend',
+        method: 'POST',
+        data: JSON.stringify({ keyword_list: [keyword], start_date, end_date, app_name }),
+    };
+
+    const res = await page.evaluate((e) => {
+        function queryData() {
+            const p = new Promise((resolve) => {
+                const req = new XMLHttpRequest();
+                req.open(e.method, e.url, true);
+                req.setRequestHeader('accept', 'application/json, text/plain, */*');
+                req.setRequestHeader('content-type', 'application/json;charset=UTF-8');
+                req.setRequestHeader('tea-uid', '7054893410171930123');
+                req.onreadystatechange = function () {
+                    if (req.readyState === 4 && req.status === 200) {
+                        resolve(req.responseText);
+                    } else {
+                        return;
+                    }
+                };
+                req.send(e.data);
+            });
+            return p;
+        }
+        return Promise.all([queryData()]).then((result) => result);
+    }, e);
+    return res[0];
+};
+
+// Decrypt Data
+const { createDecipheriv } = require('node:crypto');
+const key = 'anN2bXA2NjYsamlh';
+const iv = 'amlheW91LHFpYW53';
+const algorithm = 'aes-128-cfb';
+const decrypt = (rawData) => {
+    const encrypted = Buffer.from(rawData, 'base64').toString('hex');
+    const decipher = createDecipheriv(algorithm, key, iv);
+    const decrypted = decipher.update(encrypted, 'hex', 'utf8') + decipher.final('utf8');
+    return JSON.parse(decrypted);
+};
+
+// Generate Feed
+const searchLinkUrls = (keyword) => [
+    `https://tophub.today/search?e=tophub&q=${keyword}`,
+    `https://www.baidu.com/s?wd=${keyword}`,
+    `https://www.google.com/search?q=${keyword}`,
+    `https://m.zhihu.com/search?type=content&q=${keyword}`,
+    `https://s.weibo.com/weibo/${keyword}`,
+    `https://www.douyin.com/search/${keyword}`,
+    `https://so.toutiao.com/search?keyword=${keyword}`,
+];
+
+const searchLinkNames = ['今日热榜', '百度', '谷歌', '知乎', '微博', '抖音', '头条'];
+
+const createContent = (keyword, queryList, queryListText) =>
+    art(path.join(__dirname, 'templates', 'content.art'), {
+        keyword,
+        queryListText,
+        queries: queryList.map((query) => ({
+            links: searchLinkUrls(encodeURIComponent(query)).map((url, index) => `<a href="${url}" rel="noopener noreferrer" target="_blank">${searchLinkNames[index]}</a>`),
+            key: query,
+        })),
+    });
+
+module.exports = async (ctx) => {
+    const now = dayjs();
+    const start_date = now.subtract(DEFAULT_FETCH_DURATION_MONTH, 'month').format('YYYYMMDD');
+    const end_date = now.format('YYYYMMDD');
+    const keyword = ctx.params.keyword;
+    if (!keyword) {
+        throw Error('Invalid keyword');
+    }
+    if (ctx.params.channel && !['douyin', 'toutiao'].includes(ctx.params.channel)) {
+        throw Error('Invalid channel。 Only support `douyin` or `toutiao`');
+    }
+
+    const channel = ctx.params.channel === 'toutiao' ? 'toutiao' : 'aweme'; // default channel is `douyin`
+    const channelName = ctx.params.channel === 'toutiao' ? '头条' : '抖音';
+
+    const link = `https://trendinsight.oceanengine.com/arithmetic-index/analysis?keyword=${keyword}&appName=${channel}`;
+
+    const item = await ctx.cache.tryGet(
+        link,
+        async () => {
+            const browser = await require('@/utils/puppeteer')();
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+            });
+            await page.goto('https://trendinsight.oceanengine.com/arithmetic-index');
+            const res = await getMultiKeywordHotTrend(page, keyword, start_date, end_date, channel);
+            browser.close();
+
+            const rawData = JSON.parse(res).data;
+            const data = decrypt(rawData).hot_list[0];
+
+            const searchAverage = parseInt(data.average.search_average);
+
+            return data.search_top_point_list.map(({ date, query_list }) => {
+                const searchIndex = parseInt(data.search_hot_list.find((listItem) => listItem.datetime === date).index);
+                const relativePercent = Math.round(((searchIndex - searchAverage) / searchAverage) * 100);
+                const icon = relativePercent > 0 ? '📈' : '📉';
+
+                const uniqueQueryList = query_list.filter((query) => query !== keyword);
+                const uniqueQueryListStr = uniqueQueryList.join(', ');
+
+                const content = createContent(keyword, [keyword, ...uniqueQueryList], uniqueQueryListStr);
+
+                return {
+                    title: `${icon} ${keyword} ${relativePercent}% | ${uniqueQueryListStr}`,
+                    author: `巨量算数 - ${channelName}算数指数`,
+                    description: content,
+                    link,
+                    pubDate: timezone(parseDate(date), +8),
+                    guid: `巨量算数 - ${channelName}算数指数 | ${keyword} - ${date}`,
+                };
+            });
+        },
+        CACHE_MAX_AGE,
+        false
+    );
+
+    ctx.state.data = {
+        title: `${keyword} - ${channelName}指数波峰`,
+        link,
+        description: `巨量算数 - ${channelName}算数指数 | 关键词: ${keyword}`,
+        language: 'zh-cn',
+        item,
+    };
+};

package/lib/v2/oceanengine/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/index/:keyword/:channel?': ['Jkker'],
+};

package/lib/v2/oceanengine/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'oceanengine.com': {
+        _name: '巨量算数',
+        trendinsight: [
+            {
+                title: '抖音/头条指数波峰',
+                docs: 'https://docs.rsshub.app/other.html#ju-liang-suan-shu-suan-shu-zhi-shu',
+                source: ['/arithmetic-index/analysis'],
+                target: (params, url) => `/oceanengine/index/${new URL(url).searchParams.get('keyword')}`,
+            },
+        ],
+    },
+};

package/lib/v2/oceanengine/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/index/:keyword/:channel?', require('./arithmeticIndex'));
+};

package/lib/v2/oceanengine/templates/content.art

@@ -0,0 +1,14 @@
+<article>
+  <h2>关键词: </h2>
+  {{queryListText}}
+  &nbsp&nbsp&nbsp&nbsp&nbsp
+  <br>
+  {{each queries q}}
+  <h2>{{q.key}}</h2>
+  <p>
+    {{each q.links l}}
+    {{@l}}&nbsp&nbsp
+    {{/each}}
+  </p>
+  {{/each}}
+</article>

package/lib/v2/openwrt/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/releases/:model?': ['DIYgod'],
+};

package/lib/v2/openwrt/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'openwrt.org': {
+        _name: 'Openwrt',
+        '.': [
+            {
+                title: 'Model Releases',
+                docs: 'https://docs.rsshub.app/program-update.html#openwrt',
+                source: '/toh/:band/:model',
+                target: '/openwrt/releases/:model',
+            },
+        ],
+    },
+};

package/lib/v2/openwrt/releases.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const brand = ctx.params.brand;
+    const model = ctx.params.model;
+    const url = `https://openwrt.org/toh/${brand}/${model}`;
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const table = $('#installation').next();
+
+    ctx.state.data = {
+        title: $('h1').text() + ' - OpenWrt Releases',
+        link: url,
+        description: $('.dw-content div.level1').text(),
+        language: 'en-US',
+        item: [
+            {
+                title: table.find('.supported_current_rel').text(),
+                link: url,
+                description: `Firmware OpenWrt Install: ${table.find('.firmware_openwrt_install_url').html()}<br><br>Firmware OpenWrt Upgrade: ${$('.firmware_openwrt_upgrade_url').html()}'`,
+            },
+        ],
+    };
+};

package/lib/v2/openwrt/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/releases/:brand/:model', require('./releases'));
+};

package/lib/v2/pincong/utils.js

@@ -4,7 +4,13 @@ const puppeteerGet = (url, cache) =>
     cache.tryGet(url, async () => {
         const browser = await require('@/utils/puppeteer')();
         const page = await browser.newPage();
-        await page.goto(url);
+        await page.setRequestInterception(true);
+        page.on('request', (request) => {
+            request.resourceType() === 'document' ? request.continue() : request.abort();
+        });
+        await page.goto(url, {
+            waitUntil: 'domcontentloaded',
+        });
         const html = await page.evaluate(() => document.documentElement.innerHTML);
         browser.close();
         return html;

package/lib/v2/pku/maintainer.js

@@ -7,4 +7,7 @@ module.exports = {
     '/nsd/gd': ['MisLink'],
     '/rccp/mzyt': ['vhxubo'],
     '/scc/recruit/:type?': ['DylanXie123'],
+    '/ss/notice': ['legr4ndk'],
+    '/ss/admission': ['legr4ndk'],
+    '/ss/pgadmin': ['legr4ndk'],
 };

package/lib/v2/pku/radar.js

@@ -126,5 +126,25 @@ module.exports = {
                 },
             },
         ],
+        ss: [
+            {
+                title: '软微-通知公告',
+                docs: 'https://docs.rsshub.app/university.html#bei-jing-da-xue',
+                source: ['/index.php/newscenter/notice', '/'],
+                target: '/pku/ss/notice',
+            },
+            {
+                title: '软微-招生通知',
+                docs: 'https://docs.rsshub.app/university.html#bei-jing-da-xue',
+                source: ['/admission/admnotice', '/'],
+                target: '/pku/ss/admission',
+            },
+            {
+                title: '软微-硕士统考招生',
+                docs: 'https://docs.rsshub.app/university.html#bei-jing-da-xue',
+                source: ['/admission/admbrochure/admission01', '/'],
+                target: '/pku/ss/pgadmin',
+            },
+        ],
     },
 };

package/lib/v2/pku/router.js

@@ -7,4 +7,7 @@ module.exports = function (router) {
     router.get('/nsd/gd', require('./nsd'));
     router.get('/rccp/mzyt', require('./rccp/mzyt'));
     router.get('/scc/recruit/:type?', require('./scc/recruit'));
+    router.get('/ss/notice', require('./ss/notice.js'));
+    router.get('/ss/admission', require('./ss/admission.js'));
+    router.get('/ss/pgadmin', require('./ss/pg_admin.js'));
 };

package/lib/v2/pku/ss/admission.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const link = 'http://www.ss.pku.edu.cn';
+const host = 'http://www.ss.pku.edu.cn/index.php/admission/admnotice';
+
+const getSingleRecord = async () => {
+    const res = await got(host);
+
+    const $ = cheerio.load(res.data);
+    const list = $('#info-list-ul').find('li');
+
+    return (
+        list &&
+        list
+            .map((index, item) => {
+                item = $(item);
+                const date = item.find('.time').text();
+                return {
+                    title: item.find('a').attr('title'),
+                    pubDate: parseDate(date),
+                    link: link + item.find('a').attr('href'),
+                };
+            })
+            .get()
+    );
+};
+
+module.exports = async (ctx) => {
+    const items = await getSingleRecord();
+    const out = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                return {
+                    title: item.title,
+                    link: item.link,
+                    description: $('.article-content').html(),
+                    pubDate: item.pubDate,
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '北大软微-招生通知',
+        description: '北京大学软件与微电子学院 - 招生通知',
+        link: host,
+        item: out,
+    };
+};

package/lib/v2/pku/ss/notice.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const link = 'http://www.ss.pku.edu.cn';
+const host = 'http://www.ss.pku.edu.cn/index.php/newscenter/notice';
+
+const getSingleRecord = async () => {
+    const res = await got(host);
+
+    const $ = cheerio.load(res.data);
+    const list = $('#info-list-ul').find('li');
+
+    return (
+        list &&
+        list
+            .map((index, item) => {
+                item = $(item);
+                const date = item.find('.time').text();
+                return {
+                    title: item.find('a').attr('title'),
+                    pubDate: parseDate(date),
+                    link: link + item.find('a').attr('href'),
+                };
+            })
+            .get()
+    );
+};
+
+module.exports = async (ctx) => {
+    const items = await getSingleRecord();
+    const out = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                return {
+                    title: item.title,
+                    link: item.link,
+                    description: $('.article-content').html(),
+                    pubDate: item.pubDate,
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '北大软微-通知公告',
+        description: '北京大学软件与微电子学院 - 通知公告',
+        link: host,
+        item: out,
+    };
+};

package/lib/v2/pku/ss/pg_admin.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const link = 'http://www.ss.pku.edu.cn';
+const host = 'http://www.ss.pku.edu.cn/index.php/admission/admbrochure/admission01';
+
+const getSingleRecord = async () => {
+    const res = await got(host);
+
+    const $ = cheerio.load(res.data);
+    const list = $('#info-list-ul').find('li');
+
+    return (
+        list &&
+        list
+            .map((index, item) => {
+                item = $(item);
+                const date = item.find('.time').text();
+                return {
+                    title: item.find('a').attr('title'),
+                    pubDate: parseDate(date),
+                    link: link + item.find('a').attr('href'),
+                };
+            })
+            .get()
+    );
+};
+
+module.exports = async (ctx) => {
+    const items = await getSingleRecord();
+    const out = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                return {
+                    title: item.title,
+                    link: item.link,
+                    description: $('.article-content').html(),
+                    pubDate: item.pubDate,
+                };
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '北大软微-硕士统考招生',
+        description: '北京大学软件与微电子学院 - 硕士统考招生通知',
+        link: host,
+        item: out,
+    };
+};

package/lib/v2/prestige-av/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/series/:mid/:sort?': ['minimalistrojan'],
+};

package/lib/v2/prestige-av/radar.js

@@ -0,0 +1,18 @@
+module.exports = {
+    'prestige-av.com': {
+        _name: 'Prestige 蚊香社',
+        '.': [
+            {
+                title: '系列作品',
+                docs: 'https://docs.rsshub.app/multimedia.html#prestige-wen-xiang-she',
+                source: ['/goods/goods_list.php'],
+                target: (_params, url) => {
+                    const link = new URL(url);
+                    if (link.searchParams.get('mode') === 'series') {
+                        return link.searchParams.has('sort') ? `/prestige-av/series/${link.searchParams.get('mid')}/${link.searchParams.get('sort')}` : `/prestige-av/series/${link.searchParams.get('mid')}`;
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/prestige-av/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/series/:mid/:sort?', require('./series'));
+};