rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/lib/v2/pku/nsd.js

@@ -0,0 +1,69 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { finishArticleItem } = require('@/utils/wechat-mp');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://nsd.pku.edu.cn/sylm/gd/';
+
+const pageType = (href) => {
+    if (!href.startsWith('http')) {
+        return 'in-site';
+    }
+    const url = new URL(href);
+    if (url.hostname === 'mp.weixin.qq.com') {
+        return 'wechat-mp';
+    } else if (url.hostname === 'news.pku.edu.cn') {
+        return 'pku-news';
+    } else {
+        return 'unknown';
+    }
+};
+
+module.exports = async (ctx) => {
+    const response = await got({ url: baseUrl, https: { rejectUnauthorized: false } });
+
+    const $ = cheerio.load(response.data);
+    const list = $('div.maincontent > ul > li')
+        .map((_index, item) => {
+            const href = $(item).find('a').attr('href');
+            const type = pageType(href);
+            return {
+                title: $(item).find('a').text().trim(),
+                link: type === 'in-site' ? baseUrl + href : href,
+                pubDate: parseDate($(item).find('span').first().text(), 'YYYY-MM-DD'),
+                type,
+            };
+        })
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) => {
+            switch (item.type) {
+                case 'wechat-mp':
+                    return finishArticleItem(ctx, item);
+                case 'pku-news':
+                    return ctx.cache.tryGet(item.link, async () => {
+                        const detailResponse = await got({ url: item.link, https: { rejectUnauthorized: false } });
+                        const content = cheerio.load(detailResponse.data);
+                        item.description = content('div.pageArticle > div.col.lf').html();
+                        return item;
+                    });
+                case 'in-site':
+                    return ctx.cache.tryGet(item.link, async () => {
+                        const detailResponse = await got({ url: item.link, https: { rejectUnauthorized: false } });
+                        const content = cheerio.load(detailResponse.data);
+                        item.description = content('div.article').html();
+                        return item;
+                    });
+                default:
+                    return item;
+            }
+        })
+    );
+
+    ctx.state.data = {
+        title: '观点 - 北京大学国家发展研究院',
+        link: baseUrl,
+        item: items,
+    };
+};

package/lib/v2/pku/radar.js

@@ -76,6 +76,14 @@ module.exports = {
                 target: '/pku/hr/:category?',
             },
         ],
+        nsd: [
+            {
+                title: '观点 - 北京大学国家发展研究院',
+                docs: 'https://docs.rsshub.app/university.html#bei-jing-da-xue',
+                source: ['/'],
+                target: '/pku/nsd/gd',
+            },
+        ],
         'www.rccp': [
             {
                 title: '每周一推 - 中国政治学研究中心',

package/lib/v2/pku/router.js

@@ -4,6 +4,7 @@ module.exports = function (router) {
     router.get('/cls/lecture', require('./cls/lecture'));
     router.get('/eecs/:type?', require('./eecs'));
     router.get('/hr/:category?', require('./hr'));
+    router.get('/nsd/gd', require('./nsd'));
     router.get('/rccp/mzyt', require('./rccp/mzyt'));
     router.get('/scc/recruit/:type?', require('./scc/recruit'));
 };

package/lib/v2/playno1/av.js

@@ -0,0 +1,41 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { cookieJar, processArticle } = require('./utils');
+const baseUrl = 'http://www.playno1.com';
+
+module.exports = async (ctx) => {
+    const { catid = '78' } = ctx.params;
+    const url = `${baseUrl}/portal.php?mod=list&catid=${catid}`;
+    const response = await got(url, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+
+    let items = $('.fire_float')
+        .toArray()
+        .filter((i) => $(i).text().length)
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('h3 a').attr('title'),
+                link: item.find('h3 a').attr('href'),
+                pubDate: timezone(parseDate(item.find('.fire_left').text()), 8),
+                author: item
+                    .find('.fire_right')
+                    .text()
+                    .match(/作者：(.*)\s*\|/)[1]
+                    .trim(),
+            };
+        });
+
+    items = await processArticle(items, ctx.cache);
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        item: items,
+        language: 'zh-TW',
+    };
+};

package/lib/v2/playno1/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/av/:catid?': ['TonyRL'],
+    '/st/:catid?': ['TonyRL'],
+};

package/lib/v2/playno1/radar.js

@@ -0,0 +1,21 @@
+module.exports = {
+    'playno1.com': {
+        _name: 'PLAYNO.1玩樂達人',
+        stno1: [
+            {
+                title: '情趣',
+                docs: 'https://docs.rsshub.app/bbs.html#playno-1-wan-le-da-ren',
+                source: ['/stno1/:catid/'],
+                target: '/playno1/st/:catid',
+            },
+        ],
+        www: [
+            {
+                title: 'AV',
+                docs: 'https://docs.rsshub.app/bbs.html#playno-1-wan-le-da-ren',
+                source: ['/portal.php'],
+                target: (_params, url) => `/playno1/av/${new URL(url).searchParams.get('catid')}`,
+            },
+        ],
+    },
+};

package/lib/v2/playno1/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/av/:catid?', require('./av'));
+    router.get('/st/:catid?', require('./st'));
+};

package/lib/v2/playno1/st.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { cookieJar, processArticle } = require('./utils');
+const baseUrl = 'http://stno1.playno1.com';
+
+module.exports = async (ctx) => {
+    const { catid = 'all' } = ctx.params;
+    const url = `${baseUrl}/stno1/${catid}/`;
+    const response = await got(url, {
+        cookieJar,
+    });
+    const $ = cheerio.load(response.data);
+
+    let items = $('.fallsBox')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.ftitle a').attr('title'),
+                link: item.find('.ftitle a').attr('href'),
+                pubDate: timezone(parseDate(item.find('.dateBox').text(), 'YYYY-MM-DD HH:mm'), 8),
+                author: item.find('.dateBox span a').eq(0).text().trim(),
+            };
+        });
+
+    items = await processArticle(items, ctx.cache);
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        item: items,
+        language: 'zh-TW',
+    };
+};

package/lib/v2/playno1/utils.js

@@ -0,0 +1,32 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { CookieJar, Cookie } = require('tough-cookie');
+const cookieJar = new CookieJar();
+const cookie = Cookie.fromJSON({
+    key: 'playno1',
+    value: 'playno1Cookie',
+    domain: 'playno1.com',
+    path: '/',
+});
+(async () => {
+    await cookieJar.setCookie(cookie, 'http://www.playno1.com/');
+})();
+
+const processArticle = (items, cache) =>
+    Promise.all(
+        items.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link, {
+                    cookieJar,
+                });
+                const content = cheerio.load(detailResponse.data);
+                item.description = content('#article_content').html();
+                return item;
+            })
+        )
+    );
+
+module.exports = {
+    cookieJar,
+    processArticle,
+};

package/lib/v2/pmthinking/index.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://pmthinking.com';
+    const currentUrl = `${rootUrl}/wp-json/b2/v1/getModulePostList`;
+
+    const response = await got({
+        method: 'post',
+        url: currentUrl,
+        json: {
+            index: 2,
+            post_paged: 1,
+        },
+    });
+
+    const $ = cheerio.load(response.data.data);
+
+    let items = $('h2 a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.author = content('.post-user-name').text();
+                item.pubDate = timezone(parseDate(content('time[itemprop="datePublished"]').attr('datetime')), +8);
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    image: content('.post-style-4-top img').attr('src'),
+                    description: content('.entry-content').html(),
+                });
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '产品沉思录 · Product Thinking',
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/pmthinking/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['nczitzk'],
+};

package/lib/v2/pmthinking/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'pmthinking.com': {
+        _name: '产品沉思录',
+        '.': [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/new-media.html#chan-pin-chen-si-lu-shou-ye',
+                source: ['/'],
+                target: '/pmthinking',
+            },
+        ],
+    },
+};

package/lib/v2/pmthinking/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index'));
+};

package/lib/v2/pmthinking/templates/description.art

@@ -0,0 +1,4 @@
+{{ if image }}
+<img src="{{ image }}">
+{{ /if }}
+{{@ description }}

package/lib/v2/prestige-av/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/series/:mid/:sort?': ['minimalistrojan'],
+};

package/lib/v2/prestige-av/radar.js

@@ -0,0 +1,18 @@
+module.exports = {
+    'prestige-av.com': {
+        _name: 'Prestige 蚊香社',
+        '.': [
+            {
+                title: '系列作品',
+                docs: 'https://docs.rsshub.app/multimedia.html#prestige-wen-xiang-she',
+                source: ['/goods/goods_list.php'],
+                target: (_params, url) => {
+                    const link = new URL(url);
+                    if (link.searchParams.get('mode') === 'series') {
+                        return link.searchParams.has('sort') ? `/prestige-av/series/${link.searchParams.get('mid')}/${link.searchParams.get('sort')}` : `/prestige-av/series/${link.searchParams.get('mid')}`;
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/prestige-av/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/series/:mid/:sort?', require('./series'));
+};

package/lib/{routes → v2}/prestige-av/series.js

@@ -7,19 +7,24 @@ module.exports = async (ctx) => {
     const sort = ctx.params.sort ?? 'near';
 
     const page = await browser.newPage();
-    const link = `https://www.prestige-av.com/goods/goods_list.php?mode=series&mid=${id}&sort=${sort}`;
+    const link = `https://www.prestige-av.com/goods/goods_list.php?mode=series&mid=${id}&count=100&sort=${sort}`;
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
     await page.goto(link);
     await page.waitForSelector('.buttons');
     await page.click('#AC');
     await page.waitForSelector('#body_goods');
-    const bodyHandle = await page.$('body');
-    const html = await page.evaluate((body) => body.innerHTML, bodyHandle);
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    browser.close();
 
     const $ = cheerio.load(html);
     const list = $('div#body_goods li');
 
     ctx.state.data = {
         title: `【Prestige】${$('div[class=search_title_layout_01]').children('h1').first().text().replace('シリーズ ▶ ', '').replace(/\s*/g, '')}`,
+        description: $('meta[name=Description]').attr('content'),
         link: `https://www.prestige-av.com/goods/goods_list.php?mode=series&mid=${id}&sort=${sort}`,
         item:
             list &&
@@ -34,6 +39,4 @@ module.exports = async (ctx) => {
                 })
                 .get(),
     };
-
-    browser.close();
 };

package/lib/v2/qipamaijia/index.js

@@ -0,0 +1,26 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const rootUrl = 'https://www.qipamaijia.com';
+
+module.exports = async (ctx) => {
+    const cate = ctx.params.cate ?? '';
+    const url = `${rootUrl}/${cate}`;
+
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const title = $('#highlight').text();
+    const items = $('div.col_l > div.block')
+        .map((_index, item) => ({
+            title: $(item).find('div.content').text(),
+            link: $(item).find('a').attr('href'),
+            description: $(item).find('div.content').html() + $(item).find('div.thumb').html(),
+        }))
+        .get();
+
+    ctx.state.data = {
+        title: `奇葩买家秀 - ${title}`,
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/qipamaijia/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:cate?': ['Fatpandac'],
+};

package/lib/v2/qipamaijia/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'qipamaijia.com': {
+        _name: '奇葩买家秀',
+        '.': [
+            {
+                title: '频道',
+                docs: 'https://docs.rsshub.app/reading.html#qi-dian-tu-shou-ding',
+                source: ['/', '/:cate'],
+                target: '/qipamaijia/:cate',
+            },
+        ],
+    },
+};

package/lib/v2/qipamaijia/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:cate?', require('./index'));
+};

package/lib/v2/reuters/common.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+    const topic = ctx.params.topic ?? (category === 'authors' ? 'reuters' : '');
+
+    const rootUrl = 'https://www.reuters.com';
+    const currentUrl = topic ? `${rootUrl}/${category}/${topic}/` : `${rootUrl}/${category}/`;
+    const response = await got(currentUrl);
+    const $ = cheerio.load(response.data);
+
+    let list = $('.media-story-card__body__3tRWy a.media-story-card__heading__eqhp9')
+        .map((_, item) => {
+            item = $(item);
+            item.find('span.visually-hidden__hidden__2qXMW').remove();
+            return {
+                title: item.text(),
+                link: rootUrl + item.prop('href'),
+            };
+        })
+        .get();
+    if (!list.length) {
+        const metadata = $('script#fusion-metadata').html();
+        const metadataObj = JSON.parse(metadata.match(/Fusion.globalContent=(\{[\s\S]*?});/)[1]);
+        const articles = metadataObj.arcResult?.articles ?? metadataObj.result?.articles ?? [];
+        list = articles.map((article) => ({
+            title: article.title,
+            link: rootUrl + article.canonical_url,
+        }));
+    }
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.title = content('meta[property="og:title"]').attr('content') || item.title;
+                item.description = content('p[data-testid="paragraph-0"]').text();
+                item.pubDate = parseDate(content('meta[name="article:published_time"]').attr('content'));
+                item.author = content('meta[name="article:author"]').attr('content');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: topic ? `Reuters - ${category} - ${topic}` : `Reuters - ${category}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/reuters/investigates.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.reuters.com';
+    const currentUrl = `${rootUrl}/investigates`;
+    const response = await got(currentUrl);
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('article.section-article-container.row')
+        .map((_, item) => ({
+            title: $(item).find('h2.subtitle').text(),
+            link: $(item).find('a.row.d-flex').prop('href'),
+        }))
+        .get();
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+                const content = cheerio.load(detailResponse.data);
+
+                item.title = content('title').text();
+                item.description = content('#paragraph-0').text();
+                item.pubDate = parseDate(content('time[itemprop="datePublished"]').attr('datetime'));
+                item.author = content('meta[property="og:article:publisher"]').attr('content');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('h1.series-subtitle').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/reuters/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/investigates': ['LyleLee'],
+    '/reuters/channel/:site/:channel': ['LyleLee'], // deprecated
+    '/reuters/theWire': ['LyleLee'], // deprecated
+    '/:category/:topic?': ['LyleLee', 'HenryQW', 'proletarius101', 'black-desk'],
+};

package/lib/v2/reuters/migration_prompt.js

@@ -0,0 +1,3 @@
+module.exports = () => {
+    throw Error('The route has been deprecated. Please refer to <a href="https://docs.rsshub.app/traditional-media.html#lu-tou-she">the docs</a> for more details.');
+};

package/lib/v2/reuters/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'reuters.com': {
+        _name: '路透社',
+        '.': [
+            {
+                title: '分类/话题/作者',
+                docs: 'https://docs.rsshub.app/traditional-media.html#lu-tou-she',
+                source: ['/:category/:topic?', '/'],
+                target: '/reuters/:category/:topic?',
+            },
+            {
+                title: '深度调查栏目',
+                docs: 'https://docs.rsshub.app/traditional-media.html#lu-tou-she',
+                source: ['/investigates'],
+                target: '/reuters/inverstigates',
+            },
+        ],
+    },
+};

package/lib/v2/reuters/router.js

@@ -0,0 +1,6 @@
+module.exports = (router) => {
+    router.get('/channel/:site/:channel', require('./migration_prompt')); // deprecated
+    router.get('/theWire', require('./migration_prompt')); // deprecated
+    router.get('/investigates', require('./investigates'));
+    router.get('/:category/:topic?', require('./common'));
+};

package/lib/v2/sciencedirect/journal.js

@@ -1,11 +1,12 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const { decodeCFEmail } = require('@/utils/cf-email');
 
 module.exports = async (ctx) => {
     const id = ctx.params.id;
 
-    const rootUrl = 'https://sciencedirect.com';
+    const rootUrl = 'https://www.sciencedirect.com';
     const currentUrl = `${rootUrl}/journal/${id}/articles-in-press`;
 
     const response = await got({
@@ -44,6 +45,12 @@ module.exports = async (ctx) => {
 
                 const content = cheerio.load(detailResponse.data);
 
+                content('a.__cf_email__').each((_, e) => {
+                    e = content(e);
+                    e.after(decodeCFEmail(e.attr('data-cfemail')));
+                    e.remove();
+                });
+
                 const abstracts = content('.Abstracts').html() ?? '';
                 const keywords = content('.Keywords').html() ?? '';