rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/lib/v2/mp4er/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:type?/:caty?/:area?/:year?/:order?', require('./index'));
+};

package/lib/v2/mysql/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/release/:version?': ['nczitzk'],
+};

package/lib/v2/mysql/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'mysql.com': {
+        _name: 'MySQL',
+        dev: [
+            {
+                title: 'Release Notes',
+                docs: 'https://docs.rsshub.app/programming.html#mysql-release-notes',
+                source: ['/'],
+                target: (params, url) => `/mysql/release/${new URL(url).toString().match(/\/mysql\/(.*?)\//)[1]}`,
+            },
+        ],
+    },
+};

package/lib/v2/mysql/release.js

@@ -0,0 +1,55 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const version = ctx.params.version ?? '8.0';
+
+    const rootUrl = 'https://dev.mysql.com';
+    const currentUrl = `${rootUrl}/doc/relnotes/mysql/${version}/en`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('dt span a')
+        .slice(1, -1)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: `${currentUrl}/${item.attr('href')}`,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('.indexterm').remove();
+                content('.titlepage').first().remove();
+                content('.itemizedlist').first().remove();
+
+                item.description = content('#docs-body .section').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/mysql/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/release/:version?', require('./release'));
+};

package/lib/v2/nbd/radar.js

@@ -6,13 +6,13 @@ module.exports = {
                 title: '分类',
                 docs: 'https://docs.rsshub.app/finance.html#mei-jing-wang',
                 source: '/',
-                targer: '/nbd',
+                target: '/nbd',
             },
             {
                 title: '重磅原创',
                 docs: 'https://docs.rsshub.app/finance.html#mei-jing-wang',
                 source: '/',
-                targer: '/nbd/daily',
+                target: '/nbd/daily',
             },
         ],
     },

package/lib/v2/ncwu/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/note': ['vuhe'],
+};

package/lib/v2/ncwu/notice.js

@@ -0,0 +1,27 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const cheerio = require('cheerio');
+const baseUrl = 'https://www.ncwu.edu.cn/xxtz.htm';
+
+module.exports = async (ctx) => {
+    const response = await got(baseUrl);
+
+    const $ = cheerio.load(response.data);
+    const list = $('div.news-item')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: `「` + item.find('a.dw').text() + `」` + item.find('a.dw').next().text(),
+                description: item.find('div.detail').text(),
+                pubDate: parseDate(item.find('div.month').text() + '-' + item.find('div.day').text(), 'YYYY-MM-DD'),
+                link: item.find('a.dw').next().attr('href'),
+            };
+        });
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: baseUrl,
+        item: list,
+    };
+};

package/lib/v2/ncwu/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'ncwu.edu.cn': {
+        _name: '华北水利水电大学',
+        '.': [
+            {
+                title: '学校通知',
+                docs: 'https://docs.rsshub.app/university.html#hua-bei-shui-li-shui-dian-da-xue',
+                source: '/xxtz.htm',
+                target: '/ncwu/notice',
+            },
+        ],
+    },
+};

package/lib/v2/ncwu/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/notice', require('./notice'));
+};

package/lib/v2/nhk/news.js

@@ -0,0 +1,41 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const baseUrl = 'https://www3.nhk.or.jp';
+const apiUrl = 'https://nwapi.nhk.jp';
+
+module.exports = async (ctx) => {
+    const { lang = 'en' } = ctx.params;
+    const { data } = await got(`${apiUrl}/nhkworld/rdnewsweb/v7b/${lang}/outline/list.json`);
+    const meta = await got(`${baseUrl}/nhkworld/common/assets/news/config/${lang}.json`);
+
+    let items = data.data.map((item) => ({
+        title: item.title,
+        description: item.description,
+        link: `${baseUrl}${item.page_url}`,
+        pubDate: parseDate(item.updated_at, 'x'),
+        id: item.id,
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(`${apiUrl}/nhkworld/rdnewsweb/v6b/${lang}/detail/${item.id}.json`);
+                item.category = Object.values(data.data.categories);
+                item.description = art(path.join(__dirname, 'templates/news.art'), {
+                    img: data.data.thumbnails,
+                    description: data.data.detail.replace('\n\n', '<br><br>'),
+                });
+                delete item.id;
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${Object.values(meta.data.config.navigation.header).find((h) => h.keyname === 'topstories').name} | NHK WORLD-JAPAN News`,
+        link: `${baseUrl}/nhkworld/${lang}/news/list/`,
+        item: items,
+    };
+};

package/lib/{routes → v2}/nhk/news_web_easy.js

@@ -1,39 +1,39 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
-    const { data } = await got.get('https://www3.nhk.or.jp/news/easy/news-list.json');
+    const { data } = await got('https://www3.nhk.or.jp/news/easy/news-list.json');
     const dates = JSON.parse(data.trim())[0];
 
-    let items = [];
-    for (const articles of Object.values(dates)) {
+    let items = Object.values(dates).reduce((acc, articles) => {
         for (const article of articles) {
-            const date = new Date(article.news_prearranged_time);
-            date.setHours(date.getHours() - 9); // Japan is +0900
-            const pub_date_utc = new Date(Date.UTC(date.getFullYear(), date.getMonth(), date.getDate(), date.getHours(), date.getMinutes(), date.getSeconds()));
+            const date = timezone(parseDate(article.news_prearranged_time), +9);
 
-            items.push({
+            acc.push({
                 title: article.title,
                 description: `<h1>${article.title_with_ruby}</h1><img src="${article.news_web_image_uri}"/><br/>`,
                 guid: article.news_id,
-                pubDate: pub_date_utc.toUTCString(),
+                pubDate: date,
                 link: `https://www3.nhk.or.jp/news/easy/${article.news_id}/${article.news_id}.html`,
             });
         }
-    }
+        return acc;
+    }, []);
 
-    items = items.sort((a, b) => b.pubDate - a.pubDate).slice(0, 10);
+    items = items.sort((a, b) => b.pubDate - a.pubDate).slice(0, ctx.query.limit ? Number(ctx.query.limit) : 30);
 
-    const promises = [];
-    const getDetail = async (item) => {
-        const { data } = await got.get(item.link);
-        const $ = cheerio.load(data);
-        item.description += $('.article-body').html();
-    };
-    for (const item of items) {
-        promises.push(getDetail(item));
-    }
-    await Promise.all(promises);
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+                item.description += $('.article-body').html();
+                return item;
+            })
+        )
+    );
 
     ctx.state.data = {
         title: 'NEWS WEB EASY',

package/lib/v2/nhk/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'nhk.or.jp': {
+        _name: 'NHK',
+        www3: [
+            {
+                title: 'News Web Easy',
+                docs: 'https://docs.rsshub.app/traditional-media.html#nhk',
+                source: ['/news/easy/', '/'],
+                target: '/nhk/news_web_easy',
+            },
+            {
+                title: 'WORLD-JAPAN - 新闻提要',
+                docs: 'https://docs.rsshub.app/traditional-media.html#nhk',
+                source: ['/nhkworld/:lang/news/list/', '/nhkworld/:lang/news/'],
+                target: '/nhk/news/:lang',
+            },
+        ],
+    },
+};

package/lib/v2/nhk/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/news/:lang?', require('./news'));
+    router.get('/news_web_easy', require('./news_web_easy'));
+};

package/lib/v2/nhk/templates/news.art

@@ -0,0 +1,5 @@
+{{ if img }}
+<img src="{{ img.large || img.middle || img.small || img.min }}" alt="{{ img.alt }}">
+<br>
+{{ /if }}
+{{@ description }}

package/lib/v2/nju/rczp.js

@@ -1,38 +1,48 @@
-const cheerio = require('cheerio');
+const got = require('@/utils/got');
 const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
+const config = require('@/config').value;
 
 module.exports = async (ctx) => {
     const type = ctx.params.type;
     const title_dict = {
-        xxfb: '信息发布',
-        jylgw: '教研类岗位',
-        gllgw: '管理岗位及其他',
+        xxfb: { title: '信息发布', channelid: '9531,9532,9533,9534,9535,9419' },
+        jylgw: { title: '教研类岗位', channelid: '9420,9421,9422,9423' },
+        gllgw: { title: '管理岗位及其他', channelid: '9424,9425,9426' },
     };
-
-    const browser = await require('@/utils/puppeteer')();
-    const page = await browser.newPage();
     const link = `https://rczp.nju.edu.cn/sylm/${type}/index.html`;
-    await page.goto(link);
 
-    const html = await page.evaluate(() => document.querySelector('div.u-list').innerHTML);
-    browser.close();
+    const data = await ctx.cache.tryGet(
+        `nju:rczp:${type}`,
+        async () => {
+            const { data } = await got.post('https://rczp.nju.edu.cn/njdx/openapi/t/info/list.do', {
+                headers: {
+                    referer: link,
+                    'x-requested-with': 'XMLHttpRequest',
+                },
+                form: {
+                    channelid: Buffer.from(title_dict[type].channelid).toString('base64'),
+                    pagesize: Buffer.from('15').toString('base64'),
+                    pageno: Buffer.from('1').toString('base64'),
+                    hasPage: Buffer.from('true').toString('base64'),
+                },
+            });
+            return data;
+        },
+        config.cache.routeExpire,
+        false
+    );
 
-    const $ = cheerio.load(html);
-    const list = $('div.item');
+    const items = data.infolist.map((item) => ({
+        title: item.title,
+        description: item.summary,
+        link: item.url,
+        pubDate: parseDate(item.releasetime, 'x'),
+        author: item.username,
+    }));
 
     ctx.state.data = {
-        title: `人才招聘-${title_dict[type]}`,
+        title: `人才招聘-${title_dict[type].title}`,
         link,
-        item: list
-            .map((index, item) => {
-                item = $(item);
-                return {
-                    title: item.find('a').attr('title'),
-                    link: item.find('a').attr('href'),
-                    pubDate: timezone(parseDate(item.find('span').first().text(), 'YYYY-MM-DD'), +8),
-                };
-            })
-            .get(),
+        item: items,
     };
 };

package/lib/v2/njust/utils.js

@@ -10,6 +10,10 @@ async function getContent(url, pptr = false) {
             //         get: () => undefined,
             //     });
             // });
+            await page.setRequestInterception(true);
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+            });
             await page.goto(url, {
                 waitUntil: 'networkidle0',
             });

package/lib/v2/nuaa/utils/pypasswaf.js

@@ -5,12 +5,11 @@ const host = 'http://aao.nuaa.edu.cn/';
  * @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.headers.cookie`即可
  */
 module.exports = async function getCookie() {
-    const browser = await require('@/utils/puppeteer')();
+    const browser = await require('@/utils/puppeteer')({ stealth: true });
     const page = await browser.newPage();
-    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36');
-    await page.evaluateOnNewDocument(() => {
-        // eslint-disable-next-line
-        Object.defineProperty(navigator, 'webdriver', { get: () => false });
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
     });
 
     await page.goto(host, {

package/lib/v2/nytimes/daily_briefing_chinese.js

@@ -1,61 +1,86 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
 
 module.exports = async (ctx) => {
-    const url = 'https://www.nytimes.com/zh-hans/series/daily-briefing-chinese';
+    const rootUrl = 'https://www.nytimes.com';
+    const currentUrl = `${rootUrl}/zh-hans/series/daily-briefing-chinese`;
+
     const response = await got({
         method: 'get',
-        url,
+        url: currentUrl,
+    });
+
+    const listData = JSON.parse(response.data.match(/<script>window\.__preloadedData = {"initialState":(.*),"config"/)[1]);
+
+    let items = [];
+    Object.keys(listData).forEach((key) => {
+        if (/^Article:/.test(key) && listData[key].url) {
+            const item = listData[key];
+            items.push({
+                link: item.url,
+                pubDate: parseDate(item.firstPublished),
+            });
+        }
     });
-    const data = response.data;
-    const $ = cheerio.load(data);
-    const post = $('.css-13mho3u .css-ye6x8s')
-        .map((index, elem) => {
-            const $item = $(elem);
-            const $link = $item.find('a');
-            const $title = $item.find('h2');
-
-            return {
-                title: $title.text(),
-                link: 'https://www.nytimes.com' + $link.attr('href'),
-            };
-        })
-        .get();
-
-    const items = await Promise.all(
-        post.map((item) =>
+
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const response = await got.get(item.link);
-                const data = response.data;
-                const arr = data.match(/"url":"https:\\u002F\\u002Fstatic(\S*)"articleLarge"/g);
-                const imageLink = arr.map((e) => JSON.parse(`{${e}}`).url);
-
-                const $ = cheerio.load(data);
-                $('.css-pncxxs.etfikam0').remove();
-                // remove 'Credit...'
-                $('.css-1ly73wi.e1tej78p0').remove();
-                $('.css-1xdhyk6.erfvjey0').each((i, e) => $(e).replaceWith(`<img src="${imageLink[i]}">`));
-
-                $('.css-1l44abu.ewdxa0s0').each((i, e) => {
-                    const figureNote = $(e)
-                        .children()
-                        .map((i, e) => $(e).text())
-                        .get();
-                    $(e).replaceWith(`<p><small>${figureNote.join(' ')}</small><p>`);
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
                 });
-                const date = $('time').attr('datetime');
 
-                item.description = $('.meteredContent.css-1r7ky0e').html();
-                item.pubDate = parseDate(date);
+                const content = cheerio.load(detailResponse.data);
+                const itemData = JSON.parse(detailResponse.data.match(/<script>window\.__preloadedData = {"initialState":(.*),"config"/)[1]);
+
+                // to remove ads
+                content('.StoryBodyCompanionColumn').last().find('p').last().remove();
+
+                const images = [];
+                Object.keys(itemData).forEach((key) => {
+                    if (/^Image:/.test(key) && itemData[key].__typename === 'Image') {
+                        let cropKey = '';
+                        const image = itemData[key];
+
+                        Object.keys(image).forEach((c) => {
+                            if (/^crops/.test(c)) {
+                                cropKey = c;
+                                return;
+                            }
+                        });
+                        images.push({
+                            caption: image.legacyHtmlCaption,
+                            url: itemData[itemData[image[cropKey][0].id].renditions[0].id].url,
+                        });
+                    }
+                });
+
+                let i = 0;
+                content('figure').each(function () {
+                    content(this).html(
+                        art(path.join(__dirname, 'templates/image.art'), {
+                            url: images[i].url,
+                            caption: images[i++].caption,
+                        })
+                    );
+                });
+
+                item.title = content('meta[property="og:title"]').attr('content');
+                item.author = content('meta[name="byl"]').attr('content').replace(/By /, '');
+                item.description = content('section[name="articleBody"]').html();
+
                 return item;
             })
         )
     );
 
     ctx.state.data = {
-        title: '纽约时报中文网|新闻简报',
-        link: url,
+        title: '新闻简报 - The New York Times',
+        link: currentUrl,
         item: items,
     };
 };

package/lib/v2/nytimes/index.js

@@ -38,7 +38,7 @@ module.exports = async (ctx) => {
 
                 try {
                     response = await ctx.cache.tryGet(`nyt: ${link}`, async () => {
-                        const response = await got.get(link);
+                        const response = await got(link);
 
                         return response.data;
                     });
@@ -46,14 +46,14 @@ module.exports = async (ctx) => {
                     dual = true;
                 } catch (error) {
                     response = await ctx.cache.tryGet(`nyt: ${item.link}`, async () => {
-                        const response = await got.get(item.link);
+                        const response = await got(item.link);
 
                         return response.data;
                     });
                 }
             } else {
                 response = await ctx.cache.tryGet(`nyt: ${item.link}`, async () => {
-                    const response = await got.get(item.link);
+                    const response = await got(item.link);
 
                     return response.data;
                 });

package/lib/v2/nytimes/maintainer.js

@@ -1,6 +1,6 @@
 module.exports = {
     '/author/:byline': ['kevinschaul'],
     '/book/:category?': ['melvinto'],
-    '/daily_briefing_chinese': ['yueyericardo'],
+    '/daily_briefing_chinese': ['yueyericardo', 'nczitzk'],
     '/:lang?': ['HenryQW'],
 };

package/lib/v2/nytimes/templates/image.art

@@ -0,0 +1,4 @@
+<figure>
+<img src="{{ url }}">
+<figcaption>{{ caption }}</figcaption>
+</figure>

package/lib/v2/nytimes/utils.js

@@ -16,7 +16,13 @@ const ProcessImage = ($, e) => {
 const PuppeterGetter = async (ctx, browser, link) => {
     const result = await ctx.cache.tryGet(`nyt: ${link}`, async () => {
         const page = await browser.newPage();
-        await page.goto(link);
+        await page.setRequestInterception(true);
+        page.on('request', (request) => {
+            request.resourceType() === 'document' ? request.continue() : request.abort();
+        });
+        await page.goto(link, {
+            waitUntil: 'domcontentloaded',
+        });
         const response = await page.evaluate(() => document.querySelector('body').innerHTML);
         return response;
     });