rsshub 1.0.0-master.fe99193 → 1.0.0-master.fe9c82a

package/lib/v2/researchgate/publications.js

@@ -0,0 +1,70 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+
+    const rootUrl = 'https://www.researchgate.net';
+    const currentUrl = `${rootUrl}/profile/${id}`;
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(currentUrl);
+    const response = await page.evaluate(() => document.documentElement.innerHTML);
+    await page.close();
+
+    const $ = cheerio.load(response);
+
+    const list = $('div[itemprop="headline"] a')
+        .toArray()
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 15)
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await page.goto(item.link);
+                const detailResponse = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
+                const content = cheerio.load(detailResponse);
+
+                item.doi = content('meta[property="citation_doi"]').attr('content');
+                item.pubDate = parseDate(content('meta[property="citation_publication_date"]').attr('content'));
+
+                const authors = [];
+
+                content('meta[property="citation_author"]').each(function () {
+                    authors.push(content(this).attr('content'));
+                });
+
+                item.author = authors.join(', ');
+
+                item.description = content('div[itemprop="description"]').html();
+
+                return item;
+            })
+        )
+    );
+
+    await browser.close();
+
+    ctx.state.data = {
+        title: `${$('meta[property="profile:username"]').attr('content')}'s Publications - ResearchGate`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/researchgate/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'researchgate.net': {
+        _name: 'ResearchGate',
+        '.': [
+            {
+                title: 'Publications',
+                docs: 'https://docs.rsshub.app/study.html#researchgate',
+                source: ['/profile/:username'],
+                target: '/researchgate/publications/:username',
+            },
+        ],
+    },
+};

package/lib/v2/researchgate/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/publications/:id', require('./publications'));
+};

package/lib/v2/rodong/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:language?': ['TonyRL'],
+};

package/lib/v2/rodong/news.js

@@ -0,0 +1,45 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const host = 'http://www.rodong.rep.kp';
+
+module.exports = async (ctx) => {
+    const { language = 'ko' } = ctx.params;
+    const link = `${host}/${language}/index.php?MkBAMkAxQA==`;
+    const { data: response } = await got(link);
+
+    const $ = cheerio.load(response);
+    const list = $('.date_news_list .row')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.media-body').text(),
+                link: `${host}/${language}/${item.find('.media-body a').attr('href')}`,
+                author: item.find('.col-sm-3').text(),
+                pubDate: parseDate(item.find('.news_date').text(), 'YYYY.M.D.'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+
+                $('.news_Title, .NewsDetail, .News_Detail, #moveNews').remove();
+                item.description = $('.col-sm-8').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('head meta[name="description"]').attr('content'),
+        link,
+        item: items,
+    };
+};

package/lib/v2/rodong/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'rodong.rep.kp': {
+        _name: '劳动新闻',
+        '.': [
+            {
+                title: '新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#lao-dong-xin-wen',
+                source: ['/cn/index.php', '/en/index.php', '/ko/index.php', '/cn', '/en', '/ko'],
+                target: '/rodong/news',
+            },
+        ],
+    },
+};

package/lib/v2/rodong/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/news/:language?', require('./news'));
+};

package/lib/v2/ruancan/category.js

@@ -0,0 +1,8 @@
+const fetchFeed = require('./utils');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+    const currentUrl = `/cat/${category}`;
+
+    ctx.state.data = await fetchFeed(ctx, currentUrl);
+};

package/lib/v2/ruancan/index.js

@@ -1,59 +1,7 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
+const fetchFeed = require('./utils');
 
 module.exports = async (ctx) => {
-    const todo = ctx.params.do ?? '';
-    const keyword = ctx.params.keyword ?? '';
+    const currentUrl = '';
 
-    const rootUrl = 'https://www.ruancan.com';
-    const currentUrl = `${rootUrl}${todo ? (todo === 'search' ? `/page/1?s=${keyword}` : `/${todo}/${keyword}`) : ''}`;
-
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-
-    const $ = cheerio.load(response.data);
-
-    let items = $('.entry-title a')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-
-            return {
-                title: item.text(),
-                link: item.attr('href'),
-            };
-        });
-
-    items = await Promise.all(
-        items.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-
-                const content = cheerio.load(detailResponse.data);
-
-                content('.entry-content .heateorSssClear').first().nextAll().remove();
-                content('.adsbygoogle, .heateorSssClear').remove();
-
-                item.description = content('.entry-content').html();
-                item.category = content('.cat-links a')
-                    .toArray()
-                    .map((a) => content(a).text());
-                item.pubDate = parseDate(content('meta[property="article:published_time"]').attr('content'));
-
-                return item;
-            })
-        )
-    );
-
-    ctx.state.data = {
-        title: $('title').text(),
-        link: currentUrl,
-        item: items,
-    };
+    ctx.state.data = await fetchFeed(ctx, currentUrl);
 };

package/lib/v2/ruancan/maintainer.js

@@ -1,6 +1,6 @@
 module.exports = {
-    '/ruancan': ['nczitzk'],
-    '/ruancan/sort/:sort?': ['nczitzk'],
-    '/ruancan/tag/:tag': ['nczitzk'],
+    '/ruancan/category/:category?': ['nczitzk'],
     '/ruancan/search/:keyword?': ['nczitzk'],
+    '/ruancan/user/:id': ['nczitzk'],
+    '/ruancan': ['nczitzk'],
 };

package/lib/v2/ruancan/radar.js

@@ -11,14 +11,20 @@ module.exports = {
             {
                 title: '分类',
                 docs: 'https://docs.rsshub.app/new-media.html#ruan-can-fen-lei',
-                source: ['/sort/:sort', '/'],
-                target: '/ruancan/sort/:sort',
+                source: ['/cat/:category', '/'],
+                target: '/ruancan/category/:category',
             },
             {
-                title: '标签',
-                docs: 'https://docs.rsshub.app/new-media.html#ruan-can-biao-qian',
-                source: ['/tag/:tag', '/'],
-                target: '/ruancan/tag/:tag',
+                title: '搜索',
+                docs: 'https://docs.rsshub.app/new-media.html#ruan-can-sou-suo',
+                source: ['/'],
+                target: (params, url) => `/ruancan/search/${new URL(url).searchParams.get('s')}`,
+            },
+            {
+                title: '用户文章',
+                docs: 'https://docs.rsshub.app/new-media.html#ruan-can-yong-hu-wen-zhang',
+                source: ['/i/:id', '/'],
+                target: '/ruancan/user/:id',
             },
         ],
     },

package/lib/v2/ruancan/router.js

@@ -1,3 +1,6 @@
 module.exports = function (router) {
-    router.get('/:do?/:keyword?', require('./index'));
+    router.get('/category/:category?', require('./category'));
+    router.get('/search/:keyword?', require('./search'));
+    router.get('/user/:id', require('./user'));
+    router.get('/', require('./index'));
 };

package/lib/v2/ruancan/search.js

@@ -0,0 +1,8 @@
+const fetchFeed = require('./utils');
+
+module.exports = async (ctx) => {
+    const keyword = ctx.params.keyword;
+    const currentUrl = `/?s=${keyword}`;
+
+    ctx.state.data = await fetchFeed(ctx, currentUrl);
+};

package/lib/v2/ruancan/user.js

@@ -0,0 +1,8 @@
+const fetchFeed = require('./utils');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id;
+    const currentUrl = `/i/${id}`;
+
+    ctx.state.data = await fetchFeed(ctx, currentUrl);
+};

package/lib/v2/ruancan/utils.js

@@ -0,0 +1,66 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx, currentUrl) => {
+    const rootUrl = 'https://www.ruancan.com';
+    currentUrl = `${rootUrl}${currentUrl}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.item-title a')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 15)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: item.attr('href'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('.entry-copyright').remove();
+
+                content('.entry-content div').each(function () {
+                    if (/^ruanc-\d+/.test(content(this).attr('id'))) {
+                        content(this).remove();
+                    }
+                });
+
+                content('figure').each(function () {
+                    content(this).html(`<img src="${content(this).find('a').attr('href')}">`);
+                });
+
+                item.description = content('.entry-content').html();
+                item.category = content('.entry-info a[rel="category tag"]')
+                    .toArray()
+                    .map((c) => content(c).text());
+                item.pubDate = parseDate(content('.entry-info .entry-date').attr('datetime'));
+
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/saraba1st/digest.js

@@ -0,0 +1,79 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const tid = ctx.params.tid;
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const res = await got('https://bbs.saraba1st.com/2b/' + tid + '.html', {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const $ = cheerio.load(res.data);
+    const title = $('head title').text().replace(/-.*/, '');
+    const list = $('#threadlisttableid tbody[id^="normalthread_"] tr');
+    const count = list
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 20)
+        .toArray()
+        .map((each) => {
+            each = $(each);
+            const floor = each.find('th.new a.s.xst').text();
+            const floorUrl = each.find('th.new a.s.xst').attr('href');
+            return {
+                title: `${title}:${floor}`,
+                link: new URL(floorUrl, 'https://bbs.saraba1st.com/2b/').href,
+                author: each.find('td.by cite').text(),
+                pubDate: timezone(parseDate(each.find('td.by em').first().text()), +8),
+            };
+        });
+
+    const resultItems = await Promise.all(
+        count.map((i) =>
+            ctx.cache.tryGet(i.link, async () => {
+                i.description = await fetchContent(i.link);
+                return i;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `Stage1 论坛 - ${title}`,
+        link: `https://bbs.saraba1st.com/2b/${tid}.html`,
+        //        item: await resultItems,
+        item: resultItems,
+    };
+};
+
+async function fetchContent(url) {
+    // Fetch the subpageinof
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const subres = await got(url, {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const subind = cheerio.load(subres.data);
+    const stubS = subind('<div>');
+    subind('#postlist')
+        .find('div[id*="post_"] ')
+        .each(function () {
+            if (subind(this).find('td[id*="postmessage_"]').length > 0) {
+                const section = art(path.join(__dirname, 'templates/digest.art'), {
+                    author: {
+                        link: subind(this).find('.pls.favatar div.authi a').attr('href'),
+                        name: subind(this).find('.pls.favatar div.authi').text(),
+                        postinfo: subind(this).find('div.authi em[id*=authorposton]').text(),
+                    },
+                    msg: subind(this).find('td[id*="postmessage_"]').html(),
+                });
+                stubS.append(section);
+            }
+        });
+
+    return stubS.html();
+}

package/lib/v2/saraba1st/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
+    '/digest/:tid': ['shinemoon'],
     '/thread/:tid': ['zengxs'],
 };

package/lib/v2/saraba1st/radar.js

@@ -3,12 +3,21 @@ module.exports = {
         _name: 'Saraba1st',
         bbs: [
             {
-                title: '帖子',
+                title: '论坛',
                 docs: 'https://docs.rsshub.app/bbs.html#saraba1st',
                 source: '/2b/:id',
                 target: (params) => {
-                    const id = params.id.includes('thread') ? params.id.split('-')[1] : '';
-                    return id ? `/saraba1st/thread/${id}` : '';
+                    let id = params.id;
+                    // For Digest
+                    if (id.match('forum') !== null) {
+                        id = id.substring(0, id.length - 5);
+                        return `/saraba1st/digest/${id}`;
+                    }
+                    // For Thread
+                    else if (id.match('thread') !== null) {
+                        id = params.id.includes('thread') ? params.id.split('-')[1] : '';
+                        return id ? `/saraba1st/thread/${id}` : '';
+                    }
                 },
             },
         ],

package/lib/v2/saraba1st/router.js

@@ -1,3 +1,4 @@
 module.exports = (router) => {
     router.get('/thread/:tid', require('./thread'));
+    router.get('/digest/:tid', require('./digest'));
 };

package/lib/v2/saraba1st/templates/digest.art

@@ -0,0 +1,5 @@
+<div class="quoted" style="background:rgba(220,220,220,0.3);">
+    <a style="text-decoration: none;" href="https://bbs.saraba1st.com/2b/{{ author.link }}">{{ author.name }}</a>
+    <span style="text-decoration: none;color:#aaa;">{{ author.postinfo }}</span>
+</div>
+<div class="content" style="margin-bottom:20px!important;">{{@ msg }}</div>

package/lib/v2/sciencedirect/journal.js

@@ -14,7 +14,7 @@ module.exports = async (ctx) => {
         url: currentUrl,
     });
 
-    const issn = response.data.match(/ISSN(\d{8})'/)[1];
+    const issn = response.data.match(/ISSN(\w{8})'/)[1];
 
     const apiUrl = `${rootUrl}/journal/${issn}/articles-in-press/articles?path=/journal/${id}/articles-in-press&title=${id}`;
 

package/lib/v2/scitation/journal.js

@@ -1,12 +1,16 @@
 const cheerio = require('cheerio');
 const { puppeteerGet, renderDesc } = require('./utils');
 const config = require('@/config').value;
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const pub = ctx.params.pub;
     const jrn = ctx.params.jrn;
     const host = `https://${pub}.scitation.org`;
     const jrnlUrl = `${host}/toc/${jrn}/current?size=all`;
+    if (!isValidHost(pub)) {
+        throw Error('Invalid pub');
+    }
 
     // use Puppeteer due to the obstacle by cloudflare challenge
     const browser = await require('@/utils/puppeteer')();

package/lib/v2/scitation/section.js

@@ -1,6 +1,7 @@
 const cheerio = require('cheerio');
 const { puppeteerGet, renderDesc } = require('./utils');
 const config = require('@/config').value;
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const pub = ctx.params.pub;
@@ -8,6 +9,9 @@ module.exports = async (ctx) => {
     const sec = ctx.params.sec.split('+').join(' ');
     const host = `https://${pub}.scitation.org`;
     const jrnlUrl = `${host}/toc/${jrn}/current?size=all`;
+    if (!isValidHost(pub)) {
+        throw Error('Invalid pub');
+    }
 
     // use Puppeteer due to the obstacle by cloudflare challenge
     const browser = await require('@/utils/puppeteer')();

package/lib/{routes → v2}/scmp/index.js

@@ -1,26 +1,25 @@
 const parser = require('@/utils/rss-parser');
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const chromeMobileUserAgent = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
 
 module.exports = async (ctx) => {
     const categoryId = ctx.params.category_id;
     const rssUrl = `https://www.scmp.com/rss/${categoryId}/feed`;
     const feed = await parser.parseURL(rssUrl);
-    const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
 
     const items = await Promise.all(
         feed.items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 // Fetch the AMP version
                 const url = item.link.replace(/^https:\/\/www\.scmp\.com/, 'https://amp.scmp.com');
-                const response = await got({
-                    url,
-                    method: 'get',
+                const response = await got(url, {
                     headers: {
                         'User-Agent': chromeMobileUserAgent,
                     },
                 });
-                const html = response.body;
+                const html = response.data;
                 const $ = cheerio.load(html);
                 const content = $('div.article-body.clearfix');
 
@@ -37,7 +36,7 @@ module.exports = async (ctx) => {
 
                 // Metadata (categories & updatedAt)
                 const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
-                const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
+                const publishedAt = item.pubDate || $('meta[itemprop="datePublished"]').attr('content');
 
                 const categories = $('meta[name="keywords"]')
                     .attr('content')
@@ -89,8 +88,8 @@ module.exports = async (ctx) => {
                 return {
                     title: item.title,
                     id: item.guid,
-                    pubDate: new Date(publishedAt).toUTCString(),
-                    updated: new Date(updatedAt).toUTCString(),
+                    pubDate: parseDate(publishedAt),
+                    updated: parseDate(updatedAt),
                     author: item.creator,
                     link: item.link,
                     summary: summary.html(),

package/lib/v2/scmp/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category_id': ['proletarius101'],
+};

package/lib/v2/scmp/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'scmp.com': {
+        _name: 'South China Morning Post',
+        '.': [
+            {
+                title: 'News',
+                docs: 'https://docs.rsshub.app/en/traditional-media.html#south-china-morning-post',
+                source: ['/rss/:category_id/feed'],
+                target: '/scmp/:category_id',
+            },
+        ],
+    },
+};

package/lib/v2/scmp/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:category_id', require('./index'));
+};

package/lib/v2/sctv/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/programme/:id?/:limit?/:isFull?': ['nczitzk'],
+};