rsshub 1.0.0-master.f7add90 → 1.0.0-master.f7bfacc

package/lib/v2/sina/finance/stock/usstock.js

@@ -0,0 +1,37 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseArticle } = require('../../utils');
+
+module.exports = async (ctx) => {
+    const { cids = '57045' } = ctx.params;
+    const { limit = '50' } = ctx.query;
+    const { data: response } = await got('https://interface.sina.cn/pc_api/public_news_data.d.json', {
+        searchParams: {
+            cids,
+            pdps: '',
+            smartFlow: '',
+            editLevel: '0,1,2,3',
+            type: 'std_news,std_slide,std_video',
+            pageSize: limit,
+            cTime: '1483200000',
+            tm: Date.now(),
+            up: '0',
+            action: '0',
+            _: new Date().getTime(),
+        },
+    });
+    const list = response.data.map((item) => ({
+        title: item.title,
+        link: item.url,
+        pubDate: parseDate(item.ctime, 'X'),
+        author: item.media,
+    }));
+
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: '美股|美股行情|美股新闻 - 新浪财经',
+        link: 'https://finance.sina.com.cn/stock/usstock/',
+        item: items,
+    };
+};

package/lib/v2/sina/maintainer.js

@@ -0,0 +1,8 @@
+module.exports = {
+    '/csj': ['xapool'],
+    '/discovery/:type': ['LogicJake'],
+    '/finance/china/:lid?': ['yubinbai'],
+    '/finance/stock/usstock/:cids?': ['TonyRL'],
+    '/rollnews/:lid?': ['xyqfer'],
+    '/sports/:type?': ['nczitzk'],
+};

package/lib/v2/sina/radar.js

@@ -0,0 +1,49 @@
+module.exports = {
+    'sina.com.cn': {
+        _name: '新浪',
+        finance: [
+            {
+                title: '财经 - 国內',
+                docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
+                source: ['/china', '/'],
+                target: '/sina/finance/china',
+            },
+            {
+                title: '美股',
+                docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
+                source: ['/stock/usstock', '/'],
+                target: '/sina/finance/stock/usstock',
+            },
+        ],
+        news: [
+            {
+                title: '滚动新闻',
+                docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
+                source: ['/roll'],
+                target: (_, url) => `/sina/rollnews/${new URL(url).hash.match(/lid=(\d+)/)[1]}`,
+            },
+        ],
+        sports: [
+            {
+                title: '体育 - 综合',
+                docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
+                source: ['/others/:type', '/:type'],
+                target: (params) => `/sina/sports/${params.type}`,
+            },
+        ],
+        tech: [
+            {
+                title: '专栏 - 创事记',
+                docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
+                source: ['/chuangshiji', '/'],
+                target: '/sina/csj',
+            },
+            {
+                title: '科技 - 科学探索',
+                docs: 'https://docs.rsshub.app/new-media.html#xin-lang',
+                source: ['/discovery', '/'],
+                target: '/sina/discovery/zx',
+            },
+        ],
+    },
+};

package/lib/v2/sina/rollnews.js

@@ -0,0 +1,31 @@
+const { getRollNewsList, parseRollNewsList, parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const map = {
+        2509: '全部',
+        2510: '国内',
+        2511: '国际',
+        2669: '社会',
+        2512: '体育',
+        2513: '娱乐',
+        2514: '军事',
+        2515: '科技',
+        2516: '财经',
+        2517: '股市',
+        2518: '美股',
+    };
+
+    const pageid = '153';
+    const { lid = '2509' } = ctx.params;
+    const { limit = '50' } = ctx.query;
+    const response = await getRollNewsList(pageid, lid, limit);
+    const list = parseRollNewsList(response.data.result.data);
+
+    const out = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `新浪${map[lid]}滚动新闻`,
+        link: `https://news.sina.com.cn/roll/#pageid=${pageid}&lid=${lid}&k=&num=${limit}&page=1`,
+        item: out,
+    };
+};

package/lib/v2/sina/router.js

@@ -0,0 +1,8 @@
+module.exports = (router) => {
+    router.get('/csj', require('./chuangshiji'));
+    router.get('/discovery/:type', require('./discovery'));
+    router.get('/finance/china/:lid?', require('./finance/china'));
+    router.get('/finance/stock/usstock/:cids?', require('./finance/stock/usstock'));
+    router.get('/rollnews/:lid?', require('./rollnews'));
+    router.get('/sports/:type?', require('./sports'));
+};

package/lib/v2/sina/sports.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { type } = ctx.params;
+
+    let currentUrl = `https://sports.sina.com.cn/others/${type}.shtml`;
+    let query = 'ul.list2 li a';
+
+    if (type === 'ufc') {
+        currentUrl = 'http://roll.sports.sina.com.cn/s_ufc_all/index.shtml';
+        query = '#d_list ul li span a';
+    } else if (type === 'winter' || type === 'horse') {
+        currentUrl = `https://sports.sina.com.cn/${type}/`;
+        query = '[class^=news-list] .list li a';
+    }
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+    const list = $(query)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: item.attr('href').replace('http://', 'https://'),
+            };
+        });
+
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `${$('title').text().split('_')[0]} - 新浪体育`,
+        description: $('meta[name="description"]').attr('content'),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/sina/templates/slide.art

@@ -0,0 +1,3 @@
+{{ each slideData.images img }}
+    <img src="{{ img.download_img }}" alt="{{ img.intro }}">
+{{ /each }}

package/lib/v2/sina/templates/video.art

@@ -0,0 +1,5 @@
+{{ if videoUrl }}
+<video controls poster="{{ poster }}" preload="none">
+    <source src="{{ videoUrl }}">
+</video>
+{{ /if }}

package/lib/v2/sina/utils.js

@@ -0,0 +1,104 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const { join } = require('path');
+
+const getRollNewsList = (pageid, lid, limit) =>
+    got('https://feed.mix.sina.com.cn/api/roll/get', {
+        headers: {
+            referer: 'https://news.sina.com.cn/',
+        },
+        searchParams: {
+            pageid,
+            lid,
+            k: '',
+            num: limit,
+            page: 1,
+            r: Math.random(),
+            _: new Date().getTime(),
+        },
+    });
+
+const parseRollNewsList = (data) =>
+    data.map((item) => ({
+        title: item.title,
+        description: item.intro,
+        link: item.url.replace('http://', 'https://'),
+        author: item.media_name,
+        pubDate: parseDate(item.intime, 'X'),
+        updated: parseDate(item.mtime, 'X'),
+    }));
+
+const parseArticle = (item, tryGet) =>
+    tryGet(item.link, async () => {
+        const detailResponse = await got(item.link);
+        const $ = cheerio.load(detailResponse.data);
+        $('#left_hzh_ad, .appendQr_wrap, .app-kaihu-qr, .tech-quotation').remove();
+
+        const metaPublishTime = $('meta[property="article:published_time"]');
+        const htmlPubDate = $('#pub_date, .date');
+        const htmlDate = htmlPubDate.length ? timezone(parseDate(htmlPubDate.text(), ['YYYY年MM月DD日 HH:mm', 'YYYY年MM月DD日HH:mm']), 8) : null;
+        const metaDate = metaPublishTime.length ? parseDate(metaPublishTime.attr('content')) : htmlDate; // 2023-05-08T08:39:31+08:00
+        item.pubDate = item.pubDate ? item.pubDate : metaDate;
+        item.author = $('meta[property="article:author"]').attr('content');
+
+        if (item.link.startsWith('https://slide.sports.sina.com.cn/') || item.link.startsWith('https://slide.tech.sina.com.cn/')) {
+            const slideData = JSON.parse(
+                $('script')
+                    .text()
+                    .match(/var slide_data = (\{.*?\})\s/)[1]
+            );
+            item.description = art(join(__dirname, 'templates/slide.art'), { slideData });
+        } else if (item.link.startsWith('https://video.sina.com.cn/')) {
+            const videoId = $('script')
+                .text()
+                .match(/video_id:'?(.*?)'?,/)[1];
+
+            const { data: videoResponse } = await got('https://api.ivideo.sina.com.cn/public/video/play', {
+                searchParams: {
+                    video_id: videoId,
+                    appver: 'V11220.210521.03',
+                    appname: 'sinaplayer_pc',
+                    applt: 'web',
+                    tags: 'sinaplayer_pc',
+                    jsonp: '',
+                    plid: 2021012801,
+                    prid: '',
+                    uid: '',
+                    tid: '',
+                    pid: 1,
+                    ran: Math.random(),
+                    r: item.link,
+                    ssid: `gusr_pc_${Date.now()}`,
+                    preload: 0,
+                    uu: '',
+                    isAuto: 1,
+                },
+            });
+
+            const videoData = videoResponse.data;
+            const poster = videoData.image;
+            const videoUrl = videoData.videos.find((v) => v.type === 'mp4').dispatch_result.url;
+            item.description = art(join(__dirname, 'templates/video.art'), { poster, videoUrl });
+            item.pubDate = parseDate(videoData.create_time, 'X');
+        } else if (item.link.startsWith('https://news.sina.com.cn/') || item.link.startsWith('https://mil.news.sina.com.cn/')) {
+            item.description = $('#article').html();
+            item.category = $('meta[name="keywords"]').attr('content').split(',');
+        } else {
+            // https://ent.sina.com.cn
+            // https://finance.sina.com.cn
+            // https://sports.sina.com.cn
+            item.description = $('#artibody').html();
+            item.category = $('#keywords').data('wbkey')?.split(',');
+        }
+
+        return item;
+    });
+
+module.exports = {
+    getRollNewsList,
+    parseRollNewsList,
+    parseArticle,
+};

package/lib/v2/sony/downloads.js

@@ -40,6 +40,7 @@ module.exports = async (ctx) => {
         description: `Sony - ${productId.toUpperCase()}`,
         item: list.map((item) => ({
             title: item.title,
+            guid: item.title + ' - ' + item.url,
             link: item.url,
             pubDate: item.pubDate,
         })),

package/lib/v2/sspai/radar.js

@@ -73,7 +73,7 @@ module.exports = {
             {
                 title: 'Shortcuts Gallery',
                 docs: 'https://docs.rsshub.app/new-media.html#shao-shu-pai-sspai',
-                source: ['', '/*tpath'],
+                source: ['/*'],
                 target: '/sspai/shortcuts',
             },
         ],

package/lib/v2/stbu/jsjxy.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const iconv = require('iconv-lite');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const gbk2utf8 = (s) => iconv.decode(s, 'gbk');
+module.exports = async (ctx) => {
+    const baseUrl = 'https://jsjxy.stbu.edu.cn/news/';
+    const { data: response } = await got(baseUrl, {
+        responseType: 'buffer',
+        https: {
+            rejectUnauthorized: false,
+        },
+    });
+    const $ = cheerio.load(gbk2utf8(response));
+    const list = $('.content dl h4')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('a').first();
+            return {
+                title: a.text(),
+                link: a.attr('href'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link, {
+                    responseType: 'buffer',
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
+                const $ = cheerio.load(gbk2utf8(response));
+                item.description = $('.content14').first().html().trim();
+                item.pubDate = timezone(parseDate($('.article .source').text().split('日期：')[1].replace('\n', '').trim()), +8);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '四川工商学院计算机学院 - 新闻动态',
+        link: baseUrl,
+        description: '四川工商学院计算机学院 - 新闻动态',
+        item: items,
+    };
+};

package/lib/v2/stbu/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/jsjxy': ['HyperCherry'],
+    '/xyxw': ['HyperCherry'],
+};

package/lib/v2/stbu/radar.js

@@ -0,0 +1,21 @@
+module.exports = {
+    'stbu.edu.cn': {
+        _name: '四川工商学院',
+        '.': [
+            {
+                title: '学院新闻',
+                docs: 'https://docs.rsshub.app/university.html#si-chuan-gong-shang-xue-yuan',
+                source: ['/html/news/xueyuan', '/'],
+                target: '/stbu/xyxw',
+            },
+        ],
+        jsjxy: [
+            {
+                title: '计算机学院通知公告',
+                docs: 'https://docs.rsshub.app/university.html#si-chuan-gong-shang-xue-yuan',
+                source: ['/news', '/'],
+                target: '/stbu/jsjxy',
+            },
+        ],
+    },
+};

package/lib/v2/stbu/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/jsjxy', require('./jsjxy'));
+    router.get('/xyxw', require('./xyxw'));
+};

package/lib/v2/stbu/xyxw.js

@@ -0,0 +1,52 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const iconv = require('iconv-lite');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const gbk2utf8 = (s) => iconv.decode(s, 'gbk');
+module.exports = async (ctx) => {
+    const baseUrl = 'https://www.stbu.edu.cn';
+    const requestUrl = `${baseUrl}/html/news/xueyuan/`;
+    const { data: response } = await got(requestUrl, {
+        responseType: 'buffer',
+        https: {
+            rejectUnauthorized: false,
+        },
+    });
+    const $ = cheerio.load(gbk2utf8(response));
+    const list = $('.style_2 .Simple_title')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('a').first();
+            return {
+                title: a.text(),
+                link: `${baseUrl}${a.attr('href')}`,
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link, {
+                    responseType: 'buffer',
+                    https: {
+                        rejectUnauthorized: false,
+                    },
+                });
+                const $ = cheerio.load(gbk2utf8(response));
+                item.description = $('.artmainl .articlemain').first().html();
+                item.pubDate = timezone(parseDate($('.artmainl .info').text().split('|')[2].split('：')[1].trim()), +8);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '四川工商学院 - 学院新闻',
+        link: requestUrl,
+        description: '四川工商学院 - 学院新闻',
+        item: items,
+    };
+};

package/lib/v2/storyfm/episodes.js

@@ -0,0 +1,63 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://storyfm.cn';
+    const currentUrl = `${rootUrl}/episodes-list/`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.e-ep')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const a = item.find('h2.e-ep__title a');
+
+            return {
+                title: a.text(),
+                link: a.attr('href'),
+                pubDate: parseDate(item.find('.whitespace-nowrap').text()),
+                enclosure_type: 'audio/mpeg',
+                enclosure_url: item.find('audio source').attr('src'),
+                itunes_item_image: item.find('.zoom-image-container-progression img').attr('src'),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.author = content('.rs-post__author').text().replace(/By/, '').trim();
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    audio: item.enclosure_url,
+                    description: content('.rs-post__content').html(),
+                });
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '故事FM',
+        link: currentUrl,
+        item: items,
+        itunes_author: '故事FM',
+        image: $('.custom-logo-link img').attr('src'),
+    };
+};

package/lib/v2/storyfm/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/episodes': ['nczitzk'],
+    '/index': ['sanmmm'],
+};

package/lib/v2/storyfm/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'storyfm.cn': {
+        _name: '故事FM',
+        '.': [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/multimedia.html#ge-shi-fm',
+                source: ['/'],
+                target: '/storyfm/index',
+            },
+            {
+                title: '播客',
+                docs: 'https://docs.rsshub.app/multimedia.html#ge-shi-fm',
+                source: ['/episodes-list', '/'],
+                target: '/storyfm/episodes',
+            },
+        ],
+    },
+};

package/lib/v2/storyfm/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/episodes', require('./episodes'));
+    router.get('/index', require('./index'));
+};

package/lib/v2/storyfm/templates/description.art

@@ -0,0 +1,6 @@
+{{ if audio }}
+<audio controls>
+    <source src="{{ audio }}">
+</audio>
+{{ /if }}
+{{@ description }}

package/lib/v2/sysu/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
     '/sysu/cse': ['Neutrino3316', 'MegrezZhu', 'nczitzk'],
+    '/ygafz/:type?': ['TonyRL'],
 };

package/lib/v2/sysu/radar.js

@@ -9,5 +9,13 @@ module.exports = {
                 target: '/sysu/cse',
             },
         ],
+        ygafz: [
+            {
+                title: '粤港澳发展研究院',
+                docs: 'https://docs.rsshub.app/universities.html#zhong-shan-da-xue',
+                source: ['/:type?'],
+                target: '/sysu/ygafz/:type?',
+            },
+        ],
     },
 };

package/lib/v2/sysu/router.js

@@ -1,3 +1,4 @@
 module.exports = function (router) {
     router.get('/cse', require('./cse'));
+    router.get('/ygafz/:type?', require('./ygafz'));
 };

package/lib/v2/sysu/ygafz.js

@@ -0,0 +1,69 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const logger = require('@/utils/logger');
+const { CookieJar } = require('tough-cookie');
+
+module.exports = async (ctx) => {
+    const { type = 'notice' } = ctx.params;
+    const baseUrl = 'https://ygafz.sysu.edu.cn';
+    const url = `${baseUrl}/${type}`;
+
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+
+    logger.debug(`Requesting ${url}`);
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+    await page.waitForSelector('[data-block-plugin-id]');
+    const response = await page.content();
+
+    const cookieJar = new CookieJar();
+    const cookies = await page.cookies();
+    cookies.reduce((jar, cookie) => {
+        jar.setCookie(`${cookie.name}=${cookie.value}`, url);
+        return jar;
+    }, cookieJar);
+
+    browser.close();
+
+    const $ = cheerio.load(response);
+    const list = $('.list-content a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('p').text(),
+                link: `${baseUrl}${item.attr('href')}`,
+                pubDate: parseDate(item.find('.date').text()), // 2023-03-22
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(item.link, {
+                    cookieJar,
+                });
+                const $ = cheerio.load(data);
+
+                item.author = $('.article-submit')
+                    .text()
+                    .match(/发布人：(.*)/)[1];
+                item.description = $('div[data-block-plugin-id="entity_field:node:body"]').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: url,
+        item: items,
+    };
+};