rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/v2/reuters/templates/description.art

@@ -0,0 +1,44 @@
+{{ if result.summary }}
+Summary:
+<ul>{{ each result.summary s }}
+<li>{{ s.description }}</li>
+{{ /each }}</ul>
+{{ /if }}
+
+{{ if result.related_content }}
+{{ each result.related_content.galleries g }}
+    {{ each g.content_elements e }}
+        {{ if e.type === 'image' }}
+            <figure><img src="{{ e.url }}" alt="{{ e.alt_text }}">
+            <figcaption>{{ e.caption }}</figcaption>
+            </figure>
+        {{ /if }}
+    {{ /each }}
+{{ /each }}
+{{ each result.related_content.images i }}
+    {{ if i.type === 'image' }}
+        <figure><img src="{{ i.url }}" alt="{{ i.alt_text }}">
+        <figcaption>{{ i.caption }}</figcaption>
+        </figure>
+    {{ /if }}
+{{ /each }}
+{{ each result.related_content.galleries v }}
+    {{ each v.content_elements c }}
+        {{ if c.type === 'video' }}
+            <video controls poster="{{ c.thumbnail.url }}">
+            <source src="{{ c.source.mp4 }}" type="video/mp4" />
+            </video>
+            {{ c.description }}
+        {{ /if }}
+    {{ /each }}
+{{ /each }}
+{{ /if }}
+
+{{ if result.content_elements }}
+{{ each result.content_elements e }}
+    {{ if e.type === 'paragraph' }}<p>{{@ e.content }}</p>{{ /if }}
+    {{ if e.type === 'header' }}<h{{ e.level }}>{{@ e.content }}</h{{ e.level }}>{{ /if }}
+{{ /each }}
+{{ /if }}
+
+{{ if result.sign_off }}<span>{{ result.sign_off }}</span>{{ /if }}

package/lib/v2/ruancan/index.js

@@ -16,19 +16,19 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('.item-title a')
-        .map((_, item) => {
+    let items = $('.entry-title a')
+        .toArray()
+        .map((item) => {
             item = $(item);
 
             return {
                 title: item.text(),
                 link: item.attr('href'),
             };
-        })
-        .get();
+        });
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',
@@ -37,9 +37,14 @@ module.exports = async (ctx) => {
 
                 const content = cheerio.load(detailResponse.data);
 
-                item.author = content('.author-name').text();
-                item.pubDate = parseDate(content('.published').attr('datetime'));
+                content('.entry-content .heateorSssClear').first().nextAll().remove();
+                content('.adsbygoogle, .heateorSssClear').remove();
+
                 item.description = content('.entry-content').html();
+                item.category = content('.cat-links a')
+                    .toArray()
+                    .map((a) => content(a).text());
+                item.pubDate = parseDate(content('meta[property="article:published_time"]').attr('content'));
 
                 return item;
             })

package/lib/v2/saraba1st/digest.js

@@ -0,0 +1,79 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const tid = ctx.params.tid;
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const res = await got('https://bbs.saraba1st.com/2b/' + tid + '.html', {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const $ = cheerio.load(res.data);
+    const title = $('head title').text().replace(/-.*/, '');
+    const list = $('#threadlisttableid tbody[id^="normalthread_"] tr');
+    const count = list
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 20)
+        .toArray()
+        .map((each) => {
+            each = $(each);
+            const floor = each.find('th.new a.s.xst').text();
+            const floorUrl = each.find('th.new a.s.xst').attr('href');
+            return {
+                title: `${title}:${floor}`,
+                link: new URL(floorUrl, 'https://bbs.saraba1st.com/2b/').href,
+                author: each.find('td.by cite').text(),
+                pubDate: timezone(parseDate(each.find('td.by em').first().text()), +8),
+            };
+        });
+
+    const resultItems = await Promise.all(
+        count.map((i) =>
+            ctx.cache.tryGet(i.link, async () => {
+                i.description = await fetchContent(i.link);
+                return i;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `Stage1 论坛 - ${title}`,
+        link: `https://bbs.saraba1st.com/2b/${tid}.html`,
+        //        item: await resultItems,
+        item: resultItems,
+    };
+};
+
+async function fetchContent(url) {
+    // Fetch the subpageinof
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const subres = await got(url, {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const subind = cheerio.load(subres.data);
+    const stubS = subind('<div>');
+    subind('#postlist')
+        .find('div[id*="post_"] ')
+        .each(function () {
+            if (subind(this).find('td[id*="postmessage_"]').length > 0) {
+                const section = art(path.join(__dirname, 'templates/digest.art'), {
+                    author: {
+                        link: subind(this).find('.pls.favatar div.authi a').attr('href'),
+                        name: subind(this).find('.pls.favatar div.authi').text(),
+                        postinfo: subind(this).find('div.authi em[id*=authorposton]').text(),
+                    },
+                    msg: subind(this).find('td[id*="postmessage_"]').html(),
+                });
+                stubS.append(section);
+            }
+        });
+
+    return stubS.html();
+}

package/lib/v2/saraba1st/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/digest/:tid': ['shinemoon'],
+    '/thread/:tid': ['zengxs'],
+};

package/lib/v2/saraba1st/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'saraba1st.com': {
+        _name: 'Saraba1st',
+        bbs: [
+            {
+                title: '论坛',
+                docs: 'https://docs.rsshub.app/bbs.html#saraba1st',
+                source: '/2b/:id',
+                target: (params) => {
+                    let id = params.id;
+                    // For Digest
+                    if (id.match('forum') !== null) {
+                        id = id.substring(0, id.length - 5);
+                        return `/saraba1st/digest/${id}`;
+                    }
+                    // For Thread
+                    else if (id.match('thread') !== null) {
+                        id = params.id.includes('thread') ? params.id.split('-')[1] : '';
+                        return id ? `/saraba1st/thread/${id}` : '';
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/saraba1st/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/thread/:tid', require('./thread'));
+    router.get('/digest/:tid', require('./digest'));
+};

package/lib/v2/saraba1st/templates/digest.art

@@ -0,0 +1,5 @@
+<div class="quoted" style="background:rgba(220,220,220,0.3);">
+    <a style="text-decoration: none;" href="https://bbs.saraba1st.com/2b/{{ author.link }}">{{ author.name }}</a>
+    <span style="text-decoration: none;color:#aaa;">{{ author.postinfo }}</span>
+</div>
+<div class="content" style="margin-bottom:20px!important;">{{@ msg }}</div>

package/lib/v2/saraba1st/thread.js

@@ -0,0 +1,63 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const queryString = require('query-string');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const tid = ctx.params.tid;
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+
+    const res = await got('https://bbs.saraba1st.com/2b/forum.php', {
+        searchParams: queryString.stringify({
+            mod: 'viewthread',
+            tid,
+            ordertype: 1,
+        }),
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+
+    const $ = cheerio.load(res.data);
+    const title = $('#thread_subject').text();
+
+    const list = $('#postlist > div[id^=post_]:not(:first-of-type)');
+    const count = [];
+
+    for (let i = 0; i < Math.min(list.length, 20); i++) {
+        count.push(i);
+    }
+    const resultItems = count.map((i) => {
+        const each = $(list[i]);
+        const floor = each.find('td.plc .pi a > em').text();
+        const floorUrl = each.find('td.plc .pi a').attr('href');
+        const contentHtml = $(each.find('td.t_f'));
+        const imgsHtml = contentHtml.find('img');
+        for (let i = 0; i < imgsHtml.length; i++) {
+            if (imgsHtml[i].attribs.src === 'https://static.saraba1st.com/image/common/none.gif') {
+                imgsHtml[i].attribs.src = imgsHtml[i].attribs.file;
+                const imgHtml = $(imgsHtml[i]);
+                imgHtml.removeAttr('zoomfile');
+                imgHtml.removeAttr('file');
+                imgHtml.removeAttr('onmouseover');
+                imgHtml.removeAttr('onclick');
+            }
+        }
+        contentHtml.find('div.aimg_tip').remove();
+        return {
+            title: `${title} #${floor}`,
+            link: new URL(floorUrl, 'https://bbs.saraba1st.com/2b/').href,
+            description: contentHtml.html(),
+            author: each.find('.authi .xw1').text(),
+            pubDate: timezone(parseDate(each.find('.authi em').text()), +8),
+        };
+    });
+
+    ctx.state.data = {
+        title: `Stage1 论坛 - ${title}`,
+        link: `https://bbs.saraba1st.com/2b/thread-${tid}-1-1.html`,
+        item: resultItems,
+    };
+};

package/lib/v2/science/cover.js

@@ -0,0 +1,69 @@
+// journals form AAAS publishing group
+//
+// science:        Science
+// sciadv:         Science Advances
+// sciimmunol:     Science Immunology
+// scirobotics:    Science Robotics
+// signaling:      Science Signaling
+// stm:            Science Translational Medicine
+
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+const { baseUrl } = require('./utils');
+
+module.exports = async (ctx) => {
+    const pageURL = `${baseUrl}/journals`;
+
+    const { data: pageResponse } = await got(pageURL, {
+        headers: {
+            cookie: 'cookiePolicy=iaccept;',
+        },
+    });
+    const $ = cheerio.load(pageResponse);
+
+    const items = $('.browse-journals .browse-journals__item')
+        .not('.partner-journals')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const name = item.find('.row h2').first().text().trim();
+            const volume = item
+                .find('.row li')
+                .eq(0)
+                .text()
+                .trim()
+                .match(/Volume (\d+)/)[1];
+            const issue = item
+                .find('.row li')
+                .eq(1)
+                .text()
+                .trim()
+                .match(/Issue (\d+)/)[1];
+            const date = item.find('.row li').eq(2).text().trim();
+            const coverUrl = `${baseUrl}${item.find('.cover-image__popup-moving-cover').attr('data-cover-src')}`;
+            const content = $('.cover-image__popup-view__caption-wrapper').html();
+            const link = $('.browse-journals__item__links a').eq(0).attr('href').replace('/current', '');
+
+            return {
+                title: `${name} | Volume ${volume} Issue ${issue}`,
+                description: art(path.join(__dirname, 'templates/cover.art'), {
+                    coverUrl,
+                    content,
+                }),
+                link: `${baseUrl}/${link}/${volume}/${issue}`,
+                pubDate: parseDate(date),
+            };
+        });
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('meta[property="og:description"]').attr('content'),
+        image: `${baseUrl}/apple-touch-icon.png`,
+        link: pageURL,
+        language: 'en-US',
+        item: items,
+    };
+};

package/lib/v2/science/current.js

@@ -0,0 +1,42 @@
+// journals form AAAS publishing group
+//
+// science:        Science
+// sciadv:         Science Advances
+// sciimmunol:     Science Immunology
+// scirobotics:    Science Robotics
+// signaling:      Science Signaling
+// stm:            Science Translational Medicine
+
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { baseUrl, fetchDesc, getItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { journal = 'science' } = ctx.params;
+    const pageURL = `${baseUrl}/toc/${journal}/current`;
+
+    const { data: pageResponse } = await got(pageURL, {
+        headers: {
+            cookie: 'cookiePolicy=iaccept;',
+        },
+    });
+    const $ = cheerio.load(pageResponse);
+    const pageTitleName = $('head > title').text();
+
+    const list = $('.toc__section .card')
+        .toArray()
+        .map((item) => getItem(item, $));
+
+    const browser = await require('@/utils/puppeteer')();
+    const items = await fetchDesc(list, browser, ctx.cache.tryGet);
+    await browser.close();
+
+    ctx.state.data = {
+        title: `${pageTitleName} | Current Issue`,
+        description: `Current Issue of ${pageTitleName}`,
+        image: `${baseUrl}/apple-touch-icon.png`,
+        link: pageURL,
+        language: 'en-US',
+        item: items,
+    };
+};

package/lib/v2/science/early.js

@@ -0,0 +1,33 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+const { baseUrl, fetchDesc, getItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { journal = 'science' } = ctx.params;
+    const pageUrl = `${baseUrl}/toc/${journal}/0/0`;
+
+    const { data: res } = await got(pageUrl, {
+        headers: {
+            cookie: 'cookiePolicy=iaccept;',
+        },
+    });
+    const $ = cheerio.load(res);
+
+    const list = $('.card-content .card-header')
+        .toArray()
+        .map((item) => getItem(item, $));
+
+    const browser = await require('@/utils/puppeteer')();
+    const items = await fetchDesc(list, browser, ctx.cache.tryGet);
+    await browser.close();
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('.body02').text().trim(),
+        image: `${baseUrl}/apple-touch-icon.png`,
+        link: pageUrl,
+        language: 'en-US',
+        item: items,
+    };
+};

package/lib/v2/science/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/cover': ['y9c', 'TonyRL'],
+    '/current/:journal?': ['y9c', 'TonyRL'],
+    '/early/:journal?': ['y9c', 'TonyRL'],
+};

package/lib/v2/science/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'science.org': {
+        _name: 'Science Magazine',
+        '.': [
+            {
+                title: '本期刊物',
+                docs: 'https://docs.rsshub.app/journal.html#science-xi-lie',
+                source: ['/journal/:journal', '/toc/:journal/current'],
+                target: '/science/current/:journal',
+            },
+            {
+                title: '封面故事',
+                docs: 'https://docs.rsshub.app/journal.html#science-xi-lie',
+                source: ['/'],
+                target: '/science/cover',
+            },
+            {
+                title: '在线发表',
+                docs: 'https://docs.rsshub.app/journal.html#science-xi-lie',
+                source: ['/journal/:journal', '/toc/:journal/0/0'],
+                target: '/science/early/:journal',
+            },
+        ],
+    },
+};

package/lib/v2/science/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/cover', require('./cover'));
+    router.get('/current/:journal?', require('./current'));
+    router.get('/early/:journal?', require('./early'));
+};

package/lib/v2/science/templates/article.art

@@ -0,0 +1,2 @@
+{{ if abs }}{{@ abs }}{{ /if }}
+{{ if content }}<br>{{@ content }}{{ /if }}

package/lib/v2/science/templates/cover.art

@@ -0,0 +1,2 @@
+<img src="{{ coverUrl }}">
+{{@ content }}

package/lib/v2/science/utils.js

@@ -0,0 +1,64 @@
+const cheerio = require('cheerio');
+const { art } = require('@/utils/render');
+const path = require('path');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.science.org';
+
+const fetchDesc = (list, browser, tryGet) =>
+    Promise.all(
+        list.map((item) =>
+            tryGet(item.link, async () => {
+                const page = await browser.newPage();
+                await page.setRequestInterception(true);
+                page.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+
+                await page.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
+                });
+                await page.waitForSelector('section#bodymatter, .news-article-content, .news-article-content--featured');
+                const res = await page.evaluate(() => document.documentElement.innerHTML);
+                await page.close();
+
+                const $ = cheerio.load(res);
+                const abstract = $('div#abstracts').html();
+                const content = $('.news-article-content--featured').length
+                    ? $('.news-article-content--featured').html()
+                    : $('.news-article-content').length
+                    ? $('.news-article-content').html()
+                    : $('.info-panel__formats a.btn__request-access').length || $('.info-panel__formats a.btn--access').length
+                    ? ''
+                    : $('section#bodymatter').html();
+
+                item.description = art(path.join(__dirname, 'templates/article.art'), {
+                    abs: abstract,
+                    content,
+                });
+
+                return item;
+            })
+        )
+    );
+
+const getItem = (item, $) => {
+    item = $(item);
+    return {
+        title: item.find('.article-title a').attr('title'),
+        link: `${baseUrl}${item.find('.article-title a').attr('href')}`,
+        doi: item.find('.article-title a').attr('href').replace('/doi/', ''),
+        pubDate: parseDate(item.find('.card-meta__item time').text()),
+        author: item
+            .find('.card-meta ul[title="list of authors"] li')
+            .toArray()
+            .map((author) => $(author).text())
+            .join(', '),
+    };
+};
+
+module.exports = {
+    baseUrl,
+    fetchDesc,
+    getItem,
+};

package/lib/v2/sciencedirect/journal.js

@@ -14,7 +14,7 @@ module.exports = async (ctx) => {
         url: currentUrl,
     });
 
-    const issn = response.data.match(/ISSN(\d{8})'/)[1];
+    const issn = response.data.match(/ISSN(\w{8})'/)[1];
 
     const apiUrl = `${rootUrl}/journal/${issn}/articles-in-press/articles?path=/journal/${id}/articles-in-press&title=${id}`;
 

package/lib/{routes → v2}/scmp/index.js

@@ -1,26 +1,25 @@
 const parser = require('@/utils/rss-parser');
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const chromeMobileUserAgent = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
 
 module.exports = async (ctx) => {
     const categoryId = ctx.params.category_id;
     const rssUrl = `https://www.scmp.com/rss/${categoryId}/feed`;
     const feed = await parser.parseURL(rssUrl);
-    const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
 
     const items = await Promise.all(
         feed.items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 // Fetch the AMP version
                 const url = item.link.replace(/^https:\/\/www\.scmp\.com/, 'https://amp.scmp.com');
-                const response = await got({
-                    url,
-                    method: 'get',
+                const response = await got(url, {
                     headers: {
                         'User-Agent': chromeMobileUserAgent,
                     },
                 });
-                const html = response.body;
+                const html = response.data;
                 const $ = cheerio.load(html);
                 const content = $('div.article-body.clearfix');
 
@@ -37,7 +36,7 @@ module.exports = async (ctx) => {
 
                 // Metadata (categories & updatedAt)
                 const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
-                const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
+                const publishedAt = item.pubDate || $('meta[itemprop="datePublished"]').attr('content');
 
                 const categories = $('meta[name="keywords"]')
                     .attr('content')
@@ -89,8 +88,8 @@ module.exports = async (ctx) => {
                 return {
                     title: item.title,
                     id: item.guid,
-                    pubDate: new Date(publishedAt).toUTCString(),
-                    updated: new Date(updatedAt).toUTCString(),
+                    pubDate: parseDate(publishedAt),
+                    updated: parseDate(updatedAt),
                     author: item.creator,
                     link: item.link,
                     summary: summary.html(),

package/lib/v2/scmp/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category_id': ['proletarius101'],
+};

package/lib/v2/scmp/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'scmp.com': {
+        _name: 'South China Morning Post',
+        '.': [
+            {
+                title: 'News',
+                docs: 'https://docs.rsshub.app/en/traditional-media.html#south-china-morning-post',
+                source: ['/rss/:category_id/feed'],
+                target: '/scmp/:category_id',
+            },
+        ],
+    },
+};

package/lib/v2/scmp/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:category_id', require('./index'));
+};

package/lib/v2/sctv/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/programme/:id?/:limit?/:isFull?': ['nczitzk'],
+};