rsshub 1.0.0-master.fe50f9c → 1.0.0-master.fe560d2

package/lib/v2/saraba1st/digest.js

@@ -0,0 +1,79 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const tid = ctx.params.tid;
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const res = await got('https://bbs.saraba1st.com/2b/' + tid + '.html', {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const $ = cheerio.load(res.data);
+    const title = $('head title').text().replace(/-.*/, '');
+    const list = $('#threadlisttableid tbody[id^="normalthread_"] tr');
+    const count = list
+        .slice(0, ctx.query.limit ? Number(ctx.query.limit) : 20)
+        .toArray()
+        .map((each) => {
+            each = $(each);
+            const floor = each.find('th.new a.s.xst').text();
+            const floorUrl = each.find('th.new a.s.xst').attr('href');
+            return {
+                title: `${title}:${floor}`,
+                link: new URL(floorUrl, 'https://bbs.saraba1st.com/2b/').href,
+                author: each.find('td.by cite').text(),
+                pubDate: timezone(parseDate(each.find('td.by em').first().text()), +8),
+            };
+        });
+
+    const resultItems = await Promise.all(
+        count.map((i) =>
+            ctx.cache.tryGet(i.link, async () => {
+                i.description = await fetchContent(i.link);
+                return i;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `Stage1 论坛 - ${title}`,
+        link: `https://bbs.saraba1st.com/2b/${tid}.html`,
+        //        item: await resultItems,
+        item: resultItems,
+    };
+};
+
+async function fetchContent(url) {
+    // Fetch the subpageinof
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+    const subres = await got(url, {
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+    const subind = cheerio.load(subres.data);
+    const stubS = subind('<div>');
+    subind('#postlist')
+        .find('div[id*="post_"] ')
+        .each(function () {
+            if (subind(this).find('td[id*="postmessage_"]').length > 0) {
+                const section = art(path.join(__dirname, 'templates/digest.art'), {
+                    author: {
+                        link: subind(this).find('.pls.favatar div.authi a').attr('href'),
+                        name: subind(this).find('.pls.favatar div.authi').text(),
+                        postinfo: subind(this).find('div.authi em[id*=authorposton]').text(),
+                    },
+                    msg: subind(this).find('td[id*="postmessage_"]').html(),
+                });
+                stubS.append(section);
+            }
+        });
+
+    return stubS.html();
+}

package/lib/v2/saraba1st/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/digest/:tid': ['shinemoon'],
+    '/thread/:tid': ['zengxs'],
+};

package/lib/v2/saraba1st/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'saraba1st.com': {
+        _name: 'Saraba1st',
+        bbs: [
+            {
+                title: '论坛',
+                docs: 'https://docs.rsshub.app/bbs.html#saraba1st',
+                source: '/2b/:id',
+                target: (params) => {
+                    let id = params.id;
+                    // For Digest
+                    if (id.match('forum') !== null) {
+                        id = id.substring(0, id.length - 5);
+                        return `/saraba1st/digest/${id}`;
+                    }
+                    // For Thread
+                    else if (id.match('thread') !== null) {
+                        id = params.id.includes('thread') ? params.id.split('-')[1] : '';
+                        return id ? `/saraba1st/thread/${id}` : '';
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/saraba1st/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/thread/:tid', require('./thread'));
+    router.get('/digest/:tid', require('./digest'));
+};

package/lib/v2/saraba1st/templates/digest.art

@@ -0,0 +1,5 @@
+<div class="quoted" style="background:rgba(220,220,220,0.3);">
+    <a style="text-decoration: none;" href="https://bbs.saraba1st.com/2b/{{ author.link }}">{{ author.name }}</a>
+    <span style="text-decoration: none;color:#aaa;">{{ author.postinfo }}</span>
+</div>
+<div class="content" style="margin-bottom:20px!important;">{{@ msg }}</div>

package/lib/v2/saraba1st/thread.js

@@ -0,0 +1,63 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const queryString = require('query-string');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const tid = ctx.params.tid;
+    const cookieString = config.saraba1st.cookie ? config.saraba1st.cookie : '';
+
+    const res = await got('https://bbs.saraba1st.com/2b/forum.php', {
+        searchParams: queryString.stringify({
+            mod: 'viewthread',
+            tid,
+            ordertype: 1,
+        }),
+        headers: {
+            Cookie: cookieString,
+        },
+    });
+
+    const $ = cheerio.load(res.data);
+    const title = $('#thread_subject').text();
+
+    const list = $('#postlist > div[id^=post_]:not(:first-of-type)');
+    const count = [];
+
+    for (let i = 0; i < Math.min(list.length, 20); i++) {
+        count.push(i);
+    }
+    const resultItems = count.map((i) => {
+        const each = $(list[i]);
+        const floor = each.find('td.plc .pi a > em').text();
+        const floorUrl = each.find('td.plc .pi a').attr('href');
+        const contentHtml = $(each.find('td.t_f'));
+        const imgsHtml = contentHtml.find('img');
+        for (let i = 0; i < imgsHtml.length; i++) {
+            if (imgsHtml[i].attribs.src === 'https://static.saraba1st.com/image/common/none.gif') {
+                imgsHtml[i].attribs.src = imgsHtml[i].attribs.file;
+                const imgHtml = $(imgsHtml[i]);
+                imgHtml.removeAttr('zoomfile');
+                imgHtml.removeAttr('file');
+                imgHtml.removeAttr('onmouseover');
+                imgHtml.removeAttr('onclick');
+            }
+        }
+        contentHtml.find('div.aimg_tip').remove();
+        return {
+            title: `${title} #${floor}`,
+            link: new URL(floorUrl, 'https://bbs.saraba1st.com/2b/').href,
+            description: contentHtml.html(),
+            author: each.find('.authi .xw1').text(),
+            pubDate: timezone(parseDate(each.find('.authi em').text()), +8),
+        };
+    });
+
+    ctx.state.data = {
+        title: `Stage1 论坛 - ${title}`,
+        link: `https://bbs.saraba1st.com/2b/thread-${tid}-1-1.html`,
+        item: resultItems,
+    };
+};

package/lib/v2/science/current.js

@@ -36,6 +36,7 @@ module.exports = async (ctx) => {
         description: `Current Issue of ${pageTitleName}`,
         image: `${baseUrl}/apple-touch-icon.png`,
         link: pageURL,
+        language: 'en-US',
         item: items,
     };
 };

package/lib/v2/science/maintainer.js

@@ -1,5 +1,5 @@
 module.exports = {
-    '/cover': ['yech1990', 'TonyRL'],
-    '/current/:journal?': ['yech1990', 'TonyRL'],
-    '/early/:journal?': ['yech1990', 'TonyRL'],
+    '/cover': ['y9c', 'TonyRL'],
+    '/current/:journal?': ['y9c', 'TonyRL'],
+    '/early/:journal?': ['y9c', 'TonyRL'],
 };

package/lib/v2/science/utils.js

@@ -18,7 +18,7 @@ const fetchDesc = (list, browser, tryGet) =>
                 await page.goto(item.link, {
                     waitUntil: 'domcontentloaded',
                 });
-                await page.waitForSelector('section#frontmatter, .news-article-content, .news-article-content--featured');
+                await page.waitForSelector('section#bodymatter, .news-article-content, .news-article-content--featured');
                 const res = await page.evaluate(() => document.documentElement.innerHTML);
                 await page.close();
 
@@ -28,7 +28,7 @@ const fetchDesc = (list, browser, tryGet) =>
                     ? $('.news-article-content--featured').html()
                     : $('.news-article-content').length
                     ? $('.news-article-content').html()
-                    : $('.info-panel__formats a.btn__request-access').length
+                    : $('.info-panel__formats a.btn__request-access').length || $('.info-panel__formats a.btn--access').length
                     ? ''
                     : $('section#bodymatter').html();
 

package/lib/{routes → v2}/scmp/index.js

@@ -1,26 +1,25 @@
 const parser = require('@/utils/rss-parser');
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const chromeMobileUserAgent = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
 
 module.exports = async (ctx) => {
     const categoryId = ctx.params.category_id;
     const rssUrl = `https://www.scmp.com/rss/${categoryId}/feed`;
     const feed = await parser.parseURL(rssUrl);
-    const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
 
     const items = await Promise.all(
         feed.items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 // Fetch the AMP version
                 const url = item.link.replace(/^https:\/\/www\.scmp\.com/, 'https://amp.scmp.com');
-                const response = await got({
-                    url,
-                    method: 'get',
+                const response = await got(url, {
                     headers: {
                         'User-Agent': chromeMobileUserAgent,
                     },
                 });
-                const html = response.body;
+                const html = response.data;
                 const $ = cheerio.load(html);
                 const content = $('div.article-body.clearfix');
 
@@ -37,7 +36,7 @@ module.exports = async (ctx) => {
 
                 // Metadata (categories & updatedAt)
                 const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
-                const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
+                const publishedAt = item.pubDate || $('meta[itemprop="datePublished"]').attr('content');
 
                 const categories = $('meta[name="keywords"]')
                     .attr('content')
@@ -89,8 +88,8 @@ module.exports = async (ctx) => {
                 return {
                     title: item.title,
                     id: item.guid,
-                    pubDate: new Date(publishedAt).toUTCString(),
-                    updated: new Date(updatedAt).toUTCString(),
+                    pubDate: parseDate(publishedAt),
+                    updated: parseDate(updatedAt),
                     author: item.creator,
                     link: item.link,
                     summary: summary.html(),

package/lib/v2/scmp/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category_id': ['proletarius101'],
+};

package/lib/v2/scmp/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'scmp.com': {
+        _name: 'South China Morning Post',
+        '.': [
+            {
+                title: 'News',
+                docs: 'https://docs.rsshub.app/en/traditional-media.html#south-china-morning-post',
+                source: ['/rss/:category_id/feed'],
+                target: '/scmp/:category_id',
+            },
+        ],
+    },
+};

package/lib/v2/scmp/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:category_id', require('./index'));
+};

package/lib/v2/sctv/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/programme/:id?/:limit?/:isFull?': ['nczitzk'],
+};

package/lib/v2/sctv/programme.js

@@ -0,0 +1,80 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? '1';
+    const limit = ctx.params.limit ? parseInt(ctx.params.limit) : 15;
+    const isFull = /t|y/i.test(ctx.params.isFull ?? 'true');
+
+    const rootUrl = 'https://www.sctv.com';
+    const apiRootUrl = 'https://kscgc.sctv-tf.com';
+    const apiUrl = `${apiRootUrl}/sctv/lookback/${id}/date.json`;
+    const listUrl = `${apiRootUrl}/sctv/lookback/index/lookbackList.json`;
+    const currentUrl = `${rootUrl}/column/detail?programmeIndex=/sctv/lookback/${id}/index.json`;
+
+    let response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let items = [];
+
+    const array = response.data.data.programmeArray.slice(0, limit).map((list) => ({
+        guid: list.id,
+        link: `${apiRootUrl}${list.programmeListUrl}`,
+    }));
+
+    await Promise.all(
+        array.map((list) =>
+            ctx.cache.tryGet(list.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: list.link,
+                });
+
+                const currentItems = detailResponse.data.data.programmeList.map((item) => ({
+                    guid: item.id,
+                    title: item.programmeTitle,
+                    link: item.programmeUrl,
+                    pubDate: timezone(parseDate(item.pubTime), +8),
+                    description: art(path.join(__dirname, 'templates/description.art'), {
+                        cover: item.programmeImage,
+                        video: item.programmeUrl,
+                    }),
+                }));
+
+                let currentFullItems = [];
+
+                if (isFull) {
+                    currentFullItems = currentItems.filter((item) => /（\d{4}\.\d{2}\.\d{2}）/.test(item.title));
+                }
+
+                items = [...items, ...(currentFullItems.length === 0 ? currentItems : currentFullItems)];
+            })
+        )
+    );
+
+    response = await got({
+        method: 'get',
+        url: listUrl,
+    });
+
+    let name, cover;
+    for (const p of response.data.data.programme_official) {
+        if (p.programmeId === id) {
+            name = p.programmeName;
+            cover = p.programmeCover;
+            break;
+        }
+    }
+
+    ctx.state.data = {
+        title: `四川广播电视台 - ${name}`,
+        link: currentUrl,
+        item: items.slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 100),
+        image: cover,
+    };
+};

package/lib/v2/sctv/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'sctv.com': {
+        _name: '四川广播电视台',
+        '.': [
+            {
+                title: '电视回放',
+                docs: 'https://docs.rsshub.app/traditional-media.html#si-chuan-guang-bo-dian-shi-tai-dian-shi-hui-fang',
+                source: ['/column/list', '/column/detail', '/'],
+                target: (params, url) => `/sctv/programme/${new URL(url).searchParams.get('programmeId')}`,
+            },
+        ],
+    },
+};

package/lib/v2/sctv/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/programme/:id?/:limit?/:isFull?', require('./programme'));
+};

package/lib/v2/sctv/templates/description.art

@@ -0,0 +1,3 @@
+<video poster="{{ cover }}" controls>
+    <source src="{{ video }}" type="video/mp4">
+</video>

package/lib/v2/shmtu/jwc.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const host = 'https://jwc.shmtu.edu.cn';
+
+async function load(link) {
+    const response = await got(link, { https: { rejectUnauthorized: false } });
+    const $ = cheerio.load(response.data);
+
+    return $('.wp_articlecontent').html();
+}
+
+const ProcessFeed = (list, caches) =>
+    Promise.all(
+        list.map((item) =>
+            caches.tryGet(item.link, async () => {
+                item.description = await load(item.link);
+                return item;
+            })
+        )
+    );
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type;
+    const info = type === 'jwgg' ? '教务公告' : '教务新闻';
+
+    const response = await got(`${host}/${type}/list.htm`, {
+        headers: {
+            Referer: host,
+        },
+        https: { rejectUnauthorized: false },
+    });
+
+    const $ = cheerio.load(response.data);
+    const list = $('tbody tr')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.views-field-nothing a').attr('title').trim(),
+                link: new URL(item.find('a').attr('href'), host).href,
+                pubDate: timezone(parseDate(item.find('.pubdate').text()), 8),
+                category: item.find('.views-field-field-xxlb').text(),
+                author: item.find('.views-field-field-xxly').text(),
+            };
+        });
+
+    const result = await ProcessFeed(list, ctx.cache);
+
+    ctx.state.data = {
+        title: `上海海事大学 ${info}`,
+        link: `${host}/${type}`,
+        description: '上海海事大学 教务信息',
+        item: result,
+    };
+};

package/lib/v2/shmtu/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/jwc/:type': ['imbytecat', 'simonsmh'],
+    '/portal/:type': ['imbytecat'],
+    '/www/:type': ['imbytecat', 'simonsmh'],
+};

package/lib/v2/shmtu/portal.js

@@ -0,0 +1,81 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const bootstrapHost = 'https://weixin.shmtu.edu.cn/dynamic/shmtuHttps';
+const host = 'https://portal.shmtu.edu.cn/api';
+
+const loadDetail = async (link) => {
+    const response = await got.post(bootstrapHost, {
+        form: {
+            interfaceUrl: link,
+        },
+        https: { rejectUnauthorized: false },
+    });
+
+    return JSON.parse(response.data);
+};
+
+const processFeed = (list, caches) =>
+    Promise.all(
+        list.map((item) =>
+            caches.tryGet(item.link, async () => {
+                const detail = await loadDetail(item.link);
+                const files = detail.field_file;
+                const images = detail.field_image;
+                item.description = art(path.join(__dirname, 'templates/portal.art'), {
+                    body: detail.body.und[0].safe_value,
+                    images: images.length !== 0 && Object.keys(images).length !== 0 ? images.und : null,
+                    files: files.length !== 0 && Object.keys(files).length !== 0 ? files.und : null,
+                });
+                item.link = detail.path;
+                return item;
+            })
+        )
+    );
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type;
+    let info;
+    switch (type) {
+        case 'bmtzgg':
+            info = '部门通知公告';
+            break;
+        case 'xsydxhdgg':
+            info = '学术与大型活动公告';
+            break;
+        case 'bmdt':
+            info = '部门动态';
+            break;
+        default:
+            info = '未知';
+            break;
+    }
+
+    const response = await got.post(bootstrapHost, {
+        form: {
+            interfaceUrl: `${host}/${type}.json?page=0`,
+        },
+        https: { rejectUnauthorized: false },
+    });
+
+    const list = JSON.parse(response.data).map((item) => ({
+        title: cheerio.load(item.title).text(),
+        link: `${host}/node/${item.nid}.json`,
+        pubDate: timezone(parseDate(item.created), 8),
+        category: item.field_department[0],
+        author: item.field_department[0],
+    }));
+
+    const result = await processFeed(list, ctx.cache);
+
+    ctx.state.data = {
+        title: `上海海事大学 ${info}`,
+        link: 'https://portal.shmtu.edu.cn/bumentongzhigonggao',
+        description: '上海海事大学 数字平台',
+        item: result,
+    };
+};

package/lib/v2/shmtu/radar.js

@@ -0,0 +1,29 @@
+module.exports = {
+    'shmtu.edu.cn': {
+        _name: '上海海事大学',
+        jwc: [
+            {
+                title: '教务信息',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:type'],
+                target: '/shmtu/jwc/:type',
+            },
+        ],
+        portal: [
+            {
+                title: '数字平台',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:type'],
+                target: '/shmtu/portal/:type',
+            },
+        ],
+        www: [
+            {
+                title: '官网信息',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:type'],
+                target: '/shmtu/www/:type',
+            },
+        ],
+    },
+};

package/lib/v2/shmtu/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/jwc/:type', require('./jwc'));
+    router.get('/portal/:type', require('./portal'));
+    router.get('/www/:type', require('./www'));
+};

package/lib/v2/shmtu/templates/portal.art

@@ -0,0 +1,30 @@
+{{@ body}}
+{{if images}}
+<span style="font-size:19px">
+  图片：
+</span>
+  {{each images image}}
+  <figure>
+      <img src="https://weixin.shmtu.org/dynamic/getFile?fid={{image.fid}}"
+        alt="{{image.filename}}"
+        style="margin: 0 auto" />
+      <figcaption style="text-align:center;font-size:19px">
+        {{image.alt}}
+      </figcaption>
+  </figure>
+  {{/each}}
+{{/if}}
+{{if files}}
+<span style="font-size:19px">
+  附件：
+</span>
+  {{each files file}}
+  <p style="text-indent:32px">
+    <span style="font-size:19px">
+      <a href="https://weixin.shmtu.org/dynamic/getFile?fid={{file.fid}}" target="_blank">
+        {{file.filename}}
+      </a>
+    </span>
+  </p>
+  {{/each}}
+{{/if}}