rsshub 1.0.0-master.f8b1c8d → 1.0.0-master.f961961

package/lib/v2/science/utils.js

@@ -18,7 +18,7 @@ const fetchDesc = (list, browser, tryGet) =>
                 await page.goto(item.link, {
                     waitUntil: 'domcontentloaded',
                 });
-                await page.waitForSelector('section#frontmatter, .news-article-content, .news-article-content--featured');
+                await page.waitForSelector('section#bodymatter, .news-article-content, .news-article-content--featured');
                 const res = await page.evaluate(() => document.documentElement.innerHTML);
                 await page.close();
 
@@ -28,7 +28,7 @@ const fetchDesc = (list, browser, tryGet) =>
                     ? $('.news-article-content--featured').html()
                     : $('.news-article-content').length
                     ? $('.news-article-content').html()
-                    : $('.info-panel__formats a.btn__request-access').length
+                    : $('.info-panel__formats a.btn__request-access').length || $('.info-panel__formats a.btn--access').length
                     ? ''
                     : $('section#bodymatter').html();
 

package/lib/{routes → v2}/scmp/index.js

@@ -1,26 +1,25 @@
 const parser = require('@/utils/rss-parser');
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const chromeMobileUserAgent = require('@/utils/rand-user-agent')({ browser: 'chrome', os: 'android', device: 'mobile' });
 
 module.exports = async (ctx) => {
     const categoryId = ctx.params.category_id;
     const rssUrl = `https://www.scmp.com/rss/${categoryId}/feed`;
     const feed = await parser.parseURL(rssUrl);
-    const chromeMobileUserAgent = 'Mozilla/5.0 (Linux; Android 7.0; SM-G892A Build/NRD90M; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/67.0.3396.87 Mobile Safari/537.36';
 
     const items = await Promise.all(
         feed.items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 // Fetch the AMP version
                 const url = item.link.replace(/^https:\/\/www\.scmp\.com/, 'https://amp.scmp.com');
-                const response = await got({
-                    url,
-                    method: 'get',
+                const response = await got(url, {
                     headers: {
                         'User-Agent': chromeMobileUserAgent,
                     },
                 });
-                const html = response.body;
+                const html = response.data;
                 const $ = cheerio.load(html);
                 const content = $('div.article-body.clearfix');
 
@@ -37,7 +36,7 @@ module.exports = async (ctx) => {
 
                 // Metadata (categories & updatedAt)
                 const updatedAt = $('meta[itemprop="dateModified"]').attr('content');
-                const publishedAt = $('meta[itemprop="datePublished"]').attr('content');
+                const publishedAt = item.pubDate || $('meta[itemprop="datePublished"]').attr('content');
 
                 const categories = $('meta[name="keywords"]')
                     .attr('content')
@@ -89,8 +88,8 @@ module.exports = async (ctx) => {
                 return {
                     title: item.title,
                     id: item.guid,
-                    pubDate: new Date(publishedAt).toUTCString(),
-                    updated: new Date(updatedAt).toUTCString(),
+                    pubDate: parseDate(publishedAt),
+                    updated: parseDate(updatedAt),
                     author: item.creator,
                     link: item.link,
                     summary: summary.html(),

package/lib/v2/scmp/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category_id': ['proletarius101'],
+};

package/lib/v2/scmp/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'scmp.com': {
+        _name: 'South China Morning Post',
+        '.': [
+            {
+                title: 'News',
+                docs: 'https://docs.rsshub.app/en/traditional-media.html#south-china-morning-post',
+                source: ['/rss/:category_id/feed'],
+                target: '/scmp/:category_id',
+            },
+        ],
+    },
+};

package/lib/v2/scmp/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:category_id', require('./index'));
+};

package/lib/v2/sctv/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/programme/:id?/:limit?/:isFull?': ['nczitzk'],
+};

package/lib/v2/sctv/programme.js

@@ -0,0 +1,80 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? '1';
+    const limit = ctx.params.limit ? parseInt(ctx.params.limit) : 15;
+    const isFull = /t|y/i.test(ctx.params.isFull ?? 'true');
+
+    const rootUrl = 'https://www.sctv.com';
+    const apiRootUrl = 'https://kscgc.sctv-tf.com';
+    const apiUrl = `${apiRootUrl}/sctv/lookback/${id}/date.json`;
+    const listUrl = `${apiRootUrl}/sctv/lookback/index/lookbackList.json`;
+    const currentUrl = `${rootUrl}/column/detail?programmeIndex=/sctv/lookback/${id}/index.json`;
+
+    let response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let items = [];
+
+    const array = response.data.data.programmeArray.slice(0, limit).map((list) => ({
+        guid: list.id,
+        link: `${apiRootUrl}${list.programmeListUrl}`,
+    }));
+
+    await Promise.all(
+        array.map((list) =>
+            ctx.cache.tryGet(list.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: list.link,
+                });
+
+                const currentItems = detailResponse.data.data.programmeList.map((item) => ({
+                    guid: item.id,
+                    title: item.programmeTitle,
+                    link: item.programmeUrl,
+                    pubDate: timezone(parseDate(item.pubTime), +8),
+                    description: art(path.join(__dirname, 'templates/description.art'), {
+                        cover: item.programmeImage,
+                        video: item.programmeUrl,
+                    }),
+                }));
+
+                let currentFullItems = [];
+
+                if (isFull) {
+                    currentFullItems = currentItems.filter((item) => /（\d{4}\.\d{2}\.\d{2}）/.test(item.title));
+                }
+
+                items = [...items, ...(currentFullItems.length === 0 ? currentItems : currentFullItems)];
+            })
+        )
+    );
+
+    response = await got({
+        method: 'get',
+        url: listUrl,
+    });
+
+    let name, cover;
+    for (const p of response.data.data.programme_official) {
+        if (p.programmeId === id) {
+            name = p.programmeName;
+            cover = p.programmeCover;
+            break;
+        }
+    }
+
+    ctx.state.data = {
+        title: `四川广播电视台 - ${name}`,
+        link: currentUrl,
+        item: items.slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 100),
+        image: cover,
+    };
+};

package/lib/v2/sctv/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'sctv.com': {
+        _name: '四川广播电视台',
+        '.': [
+            {
+                title: '电视回放',
+                docs: 'https://docs.rsshub.app/traditional-media.html#si-chuan-guang-bo-dian-shi-tai-dian-shi-hui-fang',
+                source: ['/column/list', '/column/detail', '/'],
+                target: (params, url) => `/sctv/programme/${new URL(url).searchParams.get('programmeId')}`,
+            },
+        ],
+    },
+};

package/lib/v2/sctv/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/programme/:id?/:limit?/:isFull?', require('./programme'));
+};

package/lib/v2/sctv/templates/description.art

@@ -0,0 +1,3 @@
+<video poster="{{ cover }}" controls>
+    <source src="{{ video }}" type="video/mp4">
+</video>

package/lib/v2/shmtu/jwc.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const host = 'https://jwc.shmtu.edu.cn';
+
+async function load(link) {
+    const response = await got(link, { https: { rejectUnauthorized: false } });
+    const $ = cheerio.load(response.data);
+
+    return $('.wp_articlecontent').html();
+}
+
+const ProcessFeed = (list, caches) =>
+    Promise.all(
+        list.map((item) =>
+            caches.tryGet(item.link, async () => {
+                item.description = await load(item.link);
+                return item;
+            })
+        )
+    );
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type;
+    const info = type === 'jwgg' ? '教务公告' : '教务新闻';
+
+    const response = await got(`${host}/${type}/list.htm`, {
+        headers: {
+            Referer: host,
+        },
+        https: { rejectUnauthorized: false },
+    });
+
+    const $ = cheerio.load(response.data);
+    const list = $('tbody tr')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.views-field-nothing a').attr('title').trim(),
+                link: new URL(item.find('a').attr('href'), host).href,
+                pubDate: timezone(parseDate(item.find('.pubdate').text()), 8),
+                category: item.find('.views-field-field-xxlb').text(),
+                author: item.find('.views-field-field-xxly').text(),
+            };
+        });
+
+    const result = await ProcessFeed(list, ctx.cache);
+
+    ctx.state.data = {
+        title: `上海海事大学 ${info}`,
+        link: `${host}/${type}`,
+        description: '上海海事大学 教务信息',
+        item: result,
+    };
+};

package/lib/v2/shmtu/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/jwc/:type': ['imbytecat', 'simonsmh'],
+    '/portal/:type': ['imbytecat'],
+    '/www/:type': ['imbytecat', 'simonsmh'],
+};

package/lib/v2/shmtu/portal.js

@@ -0,0 +1,81 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const path = require('path');
+const { art } = require('@/utils/render');
+
+const bootstrapHost = 'https://weixin.shmtu.edu.cn/dynamic/shmtuHttps';
+const host = 'https://portal.shmtu.edu.cn/api';
+
+const loadDetail = async (link) => {
+    const response = await got.post(bootstrapHost, {
+        form: {
+            interfaceUrl: link,
+        },
+        https: { rejectUnauthorized: false },
+    });
+
+    return JSON.parse(response.data);
+};
+
+const processFeed = (list, caches) =>
+    Promise.all(
+        list.map((item) =>
+            caches.tryGet(item.link, async () => {
+                const detail = await loadDetail(item.link);
+                const files = detail.field_file;
+                const images = detail.field_image;
+                item.description = art(path.join(__dirname, 'templates/portal.art'), {
+                    body: detail.body.und[0].safe_value,
+                    images: images.length !== 0 && Object.keys(images).length !== 0 ? images.und : null,
+                    files: files.length !== 0 && Object.keys(files).length !== 0 ? files.und : null,
+                });
+                item.link = detail.path;
+                return item;
+            })
+        )
+    );
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type;
+    let info;
+    switch (type) {
+        case 'bmtzgg':
+            info = '部门通知公告';
+            break;
+        case 'xsydxhdgg':
+            info = '学术与大型活动公告';
+            break;
+        case 'bmdt':
+            info = '部门动态';
+            break;
+        default:
+            info = '未知';
+            break;
+    }
+
+    const response = await got.post(bootstrapHost, {
+        form: {
+            interfaceUrl: `${host}/${type}.json?page=0`,
+        },
+        https: { rejectUnauthorized: false },
+    });
+
+    const list = JSON.parse(response.data).map((item) => ({
+        title: cheerio.load(item.title).text(),
+        link: `${host}/node/${item.nid}.json`,
+        pubDate: timezone(parseDate(item.created), 8),
+        category: item.field_department[0],
+        author: item.field_department[0],
+    }));
+
+    const result = await processFeed(list, ctx.cache);
+
+    ctx.state.data = {
+        title: `上海海事大学 ${info}`,
+        link: 'https://portal.shmtu.edu.cn/bumentongzhigonggao',
+        description: '上海海事大学 数字平台',
+        item: result,
+    };
+};

package/lib/v2/shmtu/radar.js

@@ -0,0 +1,29 @@
+module.exports = {
+    'shmtu.edu.cn': {
+        _name: '上海海事大学',
+        jwc: [
+            {
+                title: '教务信息',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:type'],
+                target: '/shmtu/jwc/:type',
+            },
+        ],
+        portal: [
+            {
+                title: '数字平台',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:type'],
+                target: '/shmtu/portal/:type',
+            },
+        ],
+        www: [
+            {
+                title: '官网信息',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:type'],
+                target: '/shmtu/www/:type',
+            },
+        ],
+    },
+};

package/lib/v2/shmtu/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/jwc/:type', require('./jwc'));
+    router.get('/portal/:type', require('./portal'));
+    router.get('/www/:type', require('./www'));
+};

package/lib/v2/shmtu/templates/portal.art

@@ -0,0 +1,30 @@
+{{@ body}}
+{{if images}}
+<span style="font-size:19px">
+  图片：
+</span>
+  {{each images image}}
+  <figure>
+      <img src="https://weixin.shmtu.org/dynamic/getFile?fid={{image.fid}}"
+        alt="{{image.filename}}"
+        style="margin: 0 auto" />
+      <figcaption style="text-align:center;font-size:19px">
+        {{image.alt}}
+      </figcaption>
+  </figure>
+  {{/each}}
+{{/if}}
+{{if files}}
+<span style="font-size:19px">
+  附件：
+</span>
+  {{each files file}}
+  <p style="text-indent:32px">
+    <span style="font-size:19px">
+      <a href="https://weixin.shmtu.org/dynamic/getFile?fid={{file.fid}}" target="_blank">
+        {{file.filename}}
+      </a>
+    </span>
+  </p>
+  {{/each}}
+{{/if}}

package/lib/v2/shmtu/www.js

@@ -0,0 +1,58 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const host = 'https://www.shmtu.edu.cn';
+
+async function load(link) {
+    const response = await got.get(link, { https: { rejectUnauthorized: false } });
+    const $ = cheerio.load(response.data);
+
+    return $('article').html();
+}
+
+const ProcessFeed = (list, caches) =>
+    Promise.all(
+        list.map((item) =>
+            caches.tryGet(item.link, async () => {
+                item.description = await load(item.link);
+
+                return item;
+            })
+        )
+    );
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type;
+    const info = type === 'notes' ? '通知公告' : '学术讲座';
+
+    const response = await got(`${host}/${type}`, {
+        headers: {
+            Referer: host,
+        },
+        https: { rejectUnauthorized: false },
+    });
+
+    const $ = cheerio.load(response.data);
+    const list = $('tbody tr')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const category = item.find('.department').text().trim();
+            return {
+                title: item.find('.title a').text().trim(),
+                link: new URL(item.find('a').attr('href'), host).href,
+                pubDate: parseDate(item.find('.date-display-single').attr('content')),
+                category,
+                author: category,
+            };
+        });
+
+    const result = await ProcessFeed(list, ctx.cache);
+
+    ctx.state.data = {
+        title: `上海海事大学 ${info}`,
+        link: `${host}/${type}`,
+        description: '上海海事大学 官网信息',
+        item: result,
+    };
+};

package/lib/v2/simpleinfo/index.js

@@ -0,0 +1,50 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const path = require('path');
+const { art } = require('@/utils/render');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const { category } = ctx.params;
+    const rootUrl = 'https://blog.simpleinfo.cc';
+    const link = `${rootUrl}${category ? (category === 'work' || category === 'talk' ? `/blog/${category}` : `/shasha77?category=${category}`) : '/shasha77'}`;
+    const response = await got(link);
+    const $ = cheerio.load(response.data);
+    const title = `${$('.-active').text()} - 簡訊設計`;
+    $('.-ad').remove();
+
+    const list = $('.article-item')
+        .map((_, item) => {
+            item = $(item);
+            return {
+                title: item.find('.title').text(),
+                link: item.find('a').first().attr('href'),
+                category: item.find('.category').text(),
+            };
+        })
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const result = await got(item.link);
+                const content = cheerio.load(result.data);
+                item.author = content('meta[property="article:author"]').attr('content');
+                item.pubDate = timezone(parseDate(content('meta[property="article:published_time"]').attr('content')), +8);
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    image: content('meta[property="og:image"]').attr('content'),
+                    description: content('.article-content').first().html(),
+                });
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title,
+        link,
+        language: 'zh-tw',
+        item: items,
+    };
+};

package/lib/v2/simpleinfo/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category?': ['haukeng'],
+};

package/lib/v2/simpleinfo/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'blog.simpleinfo.cc': {
+        _name: '簡訊設計',
+        '.': [
+            {
+                title: '志祺七七',
+                docs: 'https://docs.rsshub.app/new-media.html#jian-xun-she-ji',
+                source: '/shasha77',
+                target: (_, url) => `/simpleinfo/${new URL(url).searchParams.get('category')}`,
+            },
+            {
+                title: '夥伴聊聊 / 專案作品',
+                docs: 'https://docs.rsshub.app/new-media.html#jian-xun-she-ji',
+                source: '/blog/:category',
+                target: '/simpleinfo/:category',
+            },
+        ],
+    },
+};

package/lib/v2/simpleinfo/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:category?', require('./index'));
+};

package/lib/v2/simpleinfo/templates/description.art

@@ -0,0 +1,6 @@
+{{ if image }}
+<img src="{{ image }}">
+{{ /if }}
+{{ if description }}
+{{@ description }}
+{{ /if }}

package/lib/v2/smzdm/article.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const link = `https://zhiyou.smzdm.com/member/${ctx.params.uid}/article/`;
+
+    const response = await got(link);
+    const $ = cheerio.load(response.data);
+    const title = $('.info-stuff-nickname').text();
+
+    const list = $('.pandect-content-stuff')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.pandect-content-title a').text(),
+                link: item.find('.pandect-content-title a').attr('href'),
+                pubDate: timezone(parseDate(item.find('.pandect-content-time').text(), ['YYYY-MM-DD', 'MM-DD HH:mm']), +8),
+            };
+        });
+
+    const out = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                item.description = $('.m-contant article').html();
+                item.pubDate = timezone(parseDate($('meta[property="og:release_date"]').attr('content'), 'YYYY-MM-DD HH:mm:ss'), 8);
+                item.author = $('meta[property="og:author"]').attr('content');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${title}-什么值得买`,
+        link,
+        item: out,
+    };
+};

package/lib/v2/smzdm/baoliao.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const link = `https://zhiyou.smzdm.com/member/${ctx.params.uid}/baoliao/`;
+
+    const response = await got(link);
+    const $ = cheerio.load(response.data);
+    const title = $('.info-stuff-nickname').text();
+
+    const list = $('.pandect-content-stuff')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.pandect-content-title a').text(),
+                link: item.find('.pandect-content-title a').attr('href'),
+                pubDate: timezone(parseDate(item.find('.pandect-content-time').text(), ['YYYY-MM-DD', 'MM-DD HH:mm']), +8),
+            };
+        });
+
+    const out = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link);
+                const $ = cheerio.load(response.data);
+                item.description = $('article.txt-detail').html();
+                item.pubDate = timezone(parseDate($('.time').first().text().trim().replace('更新时间：', '')), 8);
+                item.author = title;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${title}的爆料 - 什么值得买`,
+        link,
+        item: out,
+    };
+};