rsshub 1.0.0-master.f944cc9 → 1.0.0-master.f97cdb2

package/lib/v2/nuaa/jwc/jwc.js

@@ -0,0 +1,82 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const getCookie = require('../utils/pypasswaf');
+const host = 'http://aao.nuaa.edu.cn/';
+
+const map = new Map([
+    ['tzgg', { title: '南京航空航天大学教务处 -- 通知公告', suffix: '8222/list.htm' }],
+    ['jxfw', { title: '南京航空航天大学教务处 -- 教学服务', suffix: '8230/list.htm' }],
+    ['xspy', { title: '南京航空航天大学教务处 -- 学生培养', suffix: '8231/list.htm' }],
+    ['jxjs', { title: '南京航空航天大学教务处 -- 教学建设', suffix: '8232/list.htm' }],
+    ['jxzy', { title: '南京航空航天大学教务处 -- 教学资源', suffix: '8233/list.htm' }],
+]);
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type;
+    const suffix = map.get(type).suffix;
+    const getDescription = Boolean(ctx.params.getDescription) || false;
+
+    const link = new URL(suffix, host).href;
+    const cookie = await getCookie();
+    const gotConfig = {
+        headers: {
+            cookie,
+        },
+    };
+    const response = await got(link, gotConfig);
+    const $ = cheerio.load(response.data);
+
+    const list = $('#wp_news_w8 ul li')
+        .slice(0, 10)
+        .map(function () {
+            const info = {
+                title: $(this).find('a').text(),
+                link: $(this).find('a').attr('href'),
+                date: $(this).find('span').text(),
+            };
+            return info;
+        })
+        .get();
+
+    const out = await Promise.all(
+        list.map((info) => {
+            const title = info.title || 'tzgg';
+            const date = info.date;
+            const itemUrl = new URL(info.link, host).href;
+
+            return ctx.cache.tryGet(itemUrl, async () => {
+                const arr = itemUrl.split('.');
+                const pageType = arr[arr.length - 1];
+
+                // 南航新 WAF 过于敏感
+                // 目前 description 需要遍历页面，会被 WAF 拦截导致无法输出
+                // 考虑换一种获取 description 的方式或者将标题当作 title。
+                let description = title;
+                if (getDescription) {
+                    description = itemUrl;
+                    if (pageType === 'htm' || pageType === 'html') {
+                        const response = await got(itemUrl, gotConfig);
+                        const $ = cheerio.load(response.data);
+                        description = $('.wp_articlecontent').html();
+                    }
+                }
+
+                const single = {
+                    title,
+                    link: itemUrl,
+                    description,
+                    pubDate: parseDate(date),
+                };
+                return single;
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: map.get(type).title,
+        link,
+        description: '南京航空航天大学教务处RSS',
+        item: out,
+    };
+};

package/lib/v2/nuaa/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/cs/:type/:getDescription?': ['LogicJake', 'Seiry', 'qrzbing'],
+    '/jwc/:type/:getDescription?': ['arcosx', 'Seiry', 'qrzbing'],
+    '/yjsy/:type?': ['junfengP', 'Seiry'],
+};

package/lib/v2/nuaa/radar.js

@@ -0,0 +1,23 @@
+module.exports = {
+    'nuaa.edu.cn': {
+        _name: '南京航空航天大学',
+        aao: [
+            {
+                title: '教务处',
+                docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue',
+            },
+        ],
+        cs: [
+            {
+                title: '计算机科学与技术学院',
+                docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue',
+            },
+        ],
+        'www.graduate': [
+            {
+                title: '研究生院',
+                docs: 'https://docs.rsshub.app/university.html#nan-jing-hang-kong-hang-tian-da-xue',
+            },
+        ],
+    },
+};

package/lib/v2/nuaa/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/cs/:type/:getDescription?', require('./cs/index'));
+    router.get('/jwc/:type/:getDescription?', require('./jwc/jwc'));
+    router.get('/yjsy/:type?', require('./yjsy/yjsy'));
+};

package/lib/v2/nuaa/utils/pypasswaf.js

@@ -0,0 +1,24 @@
+const host = 'http://aao.nuaa.edu.cn/';
+
+/**
+ * async function 获取cookie
+ * @desc 返回一个可用的cookie，使用 `got` 发起请求的时候，传入到`options.headers.cookie`即可
+ */
+module.exports = async function getCookie() {
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36');
+    await page.evaluateOnNewDocument(() => {
+        // eslint-disable-next-line
+        Object.defineProperty(navigator, 'webdriver', { get: () => false });
+    });
+
+    await page.goto(host, {
+        waitUntil: 'networkidle0',
+    });
+
+    let cookie = await page.cookies();
+    cookie = cookie.map(({ name, value }) => `${name}=${value}`).join('; ');
+    await browser.close();
+    return cookie;
+};

package/lib/v2/nuaa/yjsy/yjsy.js

@@ -0,0 +1,55 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const host = 'http://www.graduate.nuaa.edu.cn/';
+const { parseDate } = require('@/utils/parse-date');
+
+const map = {
+    latest: '2146/list.htm',
+    yyxw: 'yyxw/list.htm',
+    sjwj: 'sjwj/list.htm',
+    glwj: '2017/list.htm',
+    xxfw: '2147/list.htm',
+};
+
+const load = (link, ctx) =>
+    ctx.cache.tryGet(link, async () => {
+        const response = await got(host + link);
+        const $ = cheerio.load(response.data);
+        const images = $('img');
+        for (let k = 0; k < images.length; k++) {
+            $(images[k]).replaceWith(`<img src="${new URL($(images[k]).attr('src'), host).href}" />`);
+        }
+        const description = $('.wp_articlecontent').html();
+        return { description };
+    });
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type ?? 'latest';
+    const response = await got({
+        method: 'get',
+        url: host + map[type],
+    });
+    const $ = cheerio.load(response.data);
+    const list = $('#news_list ul.news_ul > li').slice(0, 10).get();
+    const process = await Promise.all(
+        list.map(async (item) => {
+            const a = $(item).find('a');
+            const t = $(item).find('span');
+            const itemUrl = a.attr('href');
+            const single = {
+                title: a.text(),
+                link: new URL(itemUrl, host).href,
+                pubDate: parseDate(t.text(), 'YYYY-MM-DD'),
+            };
+            const other = await load(itemUrl, ctx);
+            return { ...single, ...other };
+        })
+    );
+
+    ctx.state.data = {
+        title: '南航研究生院',
+        link: host,
+        description: '南航研究生院RSS',
+        item: process,
+    };
+};

package/lib/v2/nytimes/author.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const byline = ctx.params.byline;
+
+    const authorUrl = `https://www.nytimes.com/by/${byline}`;
+
+    const response = await got({
+        method: 'get',
+        url: authorUrl,
+    });
+    const data = response.data;
+    const $ = cheerio.load(data);
+
+    const authorName = $('h1').text();
+
+    const items = $('#stream-panel li')
+        .map((index, elem) => {
+            const $item = $(elem);
+            const $info = $item.find('div > div');
+            const title = $info.find('h2').text();
+            const href = $info.find('a').attr('href');
+            const link = new URL(href, authorUrl);
+            const description = $info.find('a > p').text();
+            const author = $info.find('a p > span').text();
+
+            return {
+                title,
+                author,
+                description,
+                link,
+            };
+        })
+        .get()
+        .filter((d) => d.title);
+
+    ctx.state.data = {
+        title: `${authorName} - The New York Times`,
+        link: authorUrl,
+        item: items,
+    };
+};

package/lib/v2/nytimes/book.js

@@ -0,0 +1,71 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const categoryList = {
+    'combined-print-and-e-book-nonfiction': '非虚构类 - 综合',
+    'hardcover-nonfiction': '非虚构类 - 精装本',
+    'paperback-nonfiction': '非虚构类 - 平装本',
+    'advice-how-to-and-miscellaneous': '工具类',
+    'combined-print-and-e-book-fiction': '虚构类 - 综合',
+    'hardcover-fiction': '虚构类 - 精装本',
+    'trade-fiction-paperback': '虚构类 - 平装本',
+    'childrens-middle-grade-hardcover': '儿童 - 中年级',
+    'picture-books': '儿童 - 绘本',
+    'series-books': '儿童 - 系列图书',
+    'young-adult-hardcover': '青少年',
+};
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'combined-print-and-e-book-nonfiction';
+
+    const url = `https://www.nytimes.com/books/best-sellers/${category}`;
+
+    let items = [];
+    let dataTitle = '';
+    if (categoryList[category]) {
+        const response = await got({
+            method: 'get',
+            url,
+        });
+        const data = response.data;
+        const $ = cheerio.load(data);
+        dataTitle = $('h1').eq(0).text();
+
+        items = $('article[itemprop=itemListElement]')
+            .map((index, elem) => {
+                const $item = $(elem);
+                const firstInfo = $item.find('p').eq(0).text();
+                const $name = $item.find('h3[itemprop=name]');
+                const author = $item.find('p[itemprop=author]').text();
+                const publisher = $item.find('p[itemprop=publisher]').text();
+                const description = $item.find('p[itemprop=description]').text();
+                const imageLink = $item.find('img[itemprop=image]').attr('src');
+                const $link = $item.find('ul[aria-label="Links to Book Retailers"]');
+                const links = $link.find('a').toArray();
+
+                let primaryLink = links.length > 0 ? $(links[0]).attr('href') : '';
+
+                for (const link of links) {
+                    const l = $(link);
+                    if (l.text() === 'Amazon') {
+                        primaryLink = l.attr('href');
+                        break;
+                    }
+                }
+
+                return {
+                    title: `${index + 1}: ${$name.text()}`,
+                    author,
+                    description: `<figure><img src="${imageLink}" alt="test"/><figcaption><span>${description}</span></figcaption></figure><br/>${firstInfo}<br/>Author: ${author}<br/>Publisher: ${publisher}`,
+                    link: primaryLink,
+                };
+            })
+            .get();
+    }
+
+    ctx.state.data = {
+        title: `The New York Times Best Sellers - ${dataTitle}`,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/nytimes/index.js

@@ -0,0 +1,110 @@
+const got = require('@/utils/got');
+const parser = require('@/utils/rss-parser');
+const utils = require('./utils');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    let { lang = '' } = ctx.params;
+    lang = lang.toLowerCase();
+
+    let title = '纽约时报中文网';
+    let rssUrl = 'https://cn.nytimes.com/rss/';
+
+    if (lang === 'dual') {
+        title += ' - 中英对照版';
+    } else if (lang === 'en') {
+        title += ' - 英文原版';
+    } else if (lang === 'traditionalchinese') {
+        title = '紐約時報中文網';
+        rssUrl = new URL('zh-hant', rssUrl).href;
+    } else if (lang === 'dual-traditionalchinese') {
+        title = '紐約時報中文網 - 中英對照版';
+        rssUrl = new URL('zh-hant', rssUrl).href;
+        lang = 'dual';
+    }
+
+    const browser = await require('@/utils/puppeteer')();
+    const feed = await parser.parseURL(rssUrl);
+    const items = await Promise.all(
+        feed.items.splice(0, 10).map(async (item) => {
+            let link = item.link;
+
+            let response,
+                hasEnVersion = false,
+                dual = false;
+
+            if (lang === 'dual') {
+                link = link.replace('/?utm_source=RSS', '') + '/dual';
+
+                try {
+                    response = await ctx.cache.tryGet(`nyt: ${link}`, async () => {
+                        const response = await got.get(link);
+
+                        return response.data;
+                    });
+
+                    dual = true;
+                } catch (error) {
+                    response = await ctx.cache.tryGet(`nyt: ${item.link}`, async () => {
+                        const response = await got.get(item.link);
+
+                        return response.data;
+                    });
+                }
+            } else {
+                response = await ctx.cache.tryGet(`nyt: ${item.link}`, async () => {
+                    const response = await got.get(item.link);
+
+                    return response.data;
+                });
+
+                if (lang === 'en') {
+                    const $ = cheerio.load(response);
+                    if ($('.dual-btn').length > 0) {
+                        hasEnVersion = true;
+                        link = $('.dual-btn a').last().attr().href;
+
+                        response = await utils.PuppeterGetter(ctx, browser, link);
+                    }
+                }
+            }
+
+            const single = {
+                title: item.title,
+                pubDate: item.pubDate,
+                link,
+                author: item['dc:creator'],
+            };
+
+            const result = utils.ProcessFeed(response, hasEnVersion);
+
+            // Match 感谢|謝.*?cn.letters@nytimes.com。
+            const ending = /感(&#x8C22|&#x8B1D);.*?cn\.letters@nytimes\.com。/g;
+
+            const matching = '<div class="article-paragraph">';
+            const formatted = '<br>' + matching;
+
+            single.description = result.description.replace(ending, '').split(matching).join(formatted);
+
+            if (hasEnVersion) {
+                single.title = result.title;
+                single.author = result.author;
+            }
+
+            if (dual) {
+                single.title = `「中英」${single.title}`;
+            }
+
+            return single;
+        })
+    );
+
+    browser.close();
+
+    ctx.state.data = {
+        title,
+        link: 'https://cn.nytimes.com',
+        description: title,
+        item: items,
+    };
+};

package/lib/v2/nytimes/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/author/:byline': ['kevinschaul'],
+    '/book/:category?': ['melvinto'],
+    '/daily_briefing_chinese': ['yueyericardo'],
+    '/:lang?': ['HenryQW'],
+};

package/lib/v2/nytimes/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'nytimes.com': {
+        _name: '纽约时报',
+        '.': [
+            {
+                title: '新闻简报',
+                docs: 'https://docs.rsshub.app/traditional-media.html#niu-yue-shi-bao',
+                source: '/zh-hans/series/daily-briefing-chinese',
+                target: '/nytimes/daily_briefing_chinese',
+            },
+            {
+                title: '畅销书排行榜',
+                docs: 'https://docs.rsshub.app/traditional-media.html#niu-yue-shi-bao',
+                source: ['/books/best-sellers/:category', '/books/best-sellers/'],
+                target: '/nytimes/book/:category',
+            },
+            {
+                title: '作者新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#niu-yue-shi-bao',
+                source: '/by/:byline',
+                target: '/nytimes/author/:byline',
+            },
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#niu-yue-shi-bao',
+                source: '/',
+                target: '/nytimes',
+            },
+        ],
+    },
+};

package/lib/v2/nytimes/router.js

@@ -0,0 +1,6 @@
+module.exports = (router) => {
+    router.get('/author/:byline', require('./author.js'));
+    router.get('/book/:category?', require('./book.js'));
+    router.get('/daily_briefing_chinese', require('./daily_briefing_chinese'));
+    router.get('/:lang?', require('./index'));
+};

package/lib/v2/odaily/activity.js

@@ -0,0 +1,48 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { rootUrl } = require('./utils');
+
+module.exports = async (ctx) => {
+    const currentUrl = `${rootUrl}/service/scheme/group/8?page=1&per_page=${ctx.query.limit ?? 25}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    let items = response.data.data.items.data.map((item) => ({
+        title: item.title,
+        link: `${rootUrl}/activity/${item.id}`,
+        pubDate: timezone(parseDate(item.published_at), +8),
+    }));
+
+    items = await Promise.all(
+        items.map(
+            async (item) =>
+                await ctx.cache.tryGet(item.link, async () => {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+
+                    const content = cheerio.load(detailResponse.data.match(/"content":"(.*)"}},"secondaryList":/)[1]);
+
+                    content('img').each(function () {
+                        content(this).attr('src', content(this).attr('src').replace(/\\"/g, ''));
+                    });
+
+                    item.description = content.html();
+
+                    return item;
+                })
+        )
+    );
+
+    ctx.state.data = {
+        title: '活动 - Odaily星球日报',
+        link: `${rootUrl}/activityPage`,
+        item: items,
+    };
+};

package/lib/v2/odaily/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/activity': ['nczitzk'],
+    '/newsflash': ['nczitzk'],
+    '/user/:id': ['nczitzk'],
+    '/:id?': ['nczitzk'],
+};

package/lib/v2/odaily/newsflash.js

@@ -0,0 +1,26 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { rootUrl } = require('./utils');
+
+module.exports = async (ctx) => {
+    const currentUrl = `${rootUrl}/api/pp/api/info-flow/newsflash_columns/newsflashes?b_id=&per_page=${ctx.query.limit ?? 100}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const items = response.data.data.items.map((item) => ({
+        title: item.title,
+        link: item.news_url,
+        pubDate: timezone(parseDate(item.published_at), +8),
+        description: `<p>${item.description}</p>`,
+    }));
+
+    ctx.state.data = {
+        title: '快讯 - Odaily星球日报',
+        link: `${rootUrl}/newsflash`,
+        item: items,
+    };
+};

package/lib/v2/odaily/post.js

@@ -0,0 +1,68 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { rootUrl } = require('./utils');
+
+const titles = {
+    280: '最新',
+    333: '新品',
+    331: 'DeFi',
+    334: 'NFT',
+    332: '存储',
+    330: '波卡',
+    297: '行情',
+    296: '活动',
+};
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? '280';
+
+    const currentUrl = `${rootUrl}/api/pp/api/app-front/feed-stream?feed_id=${id}&b_id=&per_page=${ctx.query.limit ?? 25}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    let items = response.data.data.items
+        .map((item) => ({
+            title: item.title,
+            author: item.user.name,
+            type: item.entity_type,
+            description: `<p>${item.summary}</p>`,
+            link: `${rootUrl}/${item.entity_type}/${item.entity_id}`,
+            pubDate: timezone(parseDate(item.published_at), +8),
+        }))
+        .filter((item) => item.type !== 'newsflash');
+
+    items = await Promise.all(
+        items.map(
+            async (item) =>
+                await ctx.cache.tryGet(item.link, async () => {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+
+                    const content = cheerio.load(detailResponse.data.match(item.type === 'post' ? /"content":"(.*)","extraction_tags":/ : /"description":"(.*?)","cover":/)[1]);
+
+                    content('img').each(function () {
+                        content(this).attr('src', content(this).attr('src').replace(/\\"/g, ''));
+                    });
+
+                    item.description = content.html();
+
+                    delete item.type;
+
+                    return item;
+                })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${titles[id]} - Odaily星球日报`,
+        link: rootUrl,
+        item: items,
+    };
+};