rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/lib/v2/shiep/radar.js

@@ -0,0 +1,181 @@
+module.exports = {
+    'shiep.edu.cn': {
+        _name: '上海电力大学',
+        dqxy: [
+            {
+                title: '电气工程学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/dqxy/:id',
+            },
+        ],
+        dxxy: [
+            {
+                title: '电子信息工程学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/dxxy/:id',
+            },
+        ],
+        energy: [
+            {
+                title: '能源与机械工程学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/energy/:id',
+            },
+        ],
+        fao: [
+            {
+                title: '国际交流与合作处（港澳台办公室）',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/fao/:id',
+            },
+        ],
+        gjjlxy: [
+            {
+                title: '国际交流学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/gjjlxy/:id',
+            },
+        ],
+        hhxy: [
+            {
+                title: '环境与化学工程学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/hhxy/:id',
+            },
+        ],
+        jgxy: [
+            {
+                title: '经济与管理学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/jgxy/:id',
+            },
+        ],
+        jjxy: [
+            {
+                title: '继续教育学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/jjxy/:id',
+            },
+        ],
+        jsjxy: [
+            {
+                title: '计算机科学与技术学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/jsjxy/:id',
+            },
+        ],
+        jwc: [
+            {
+                title: '教务处',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/jwc/:id',
+            },
+        ],
+        kyc: [
+            {
+                title: '科研处',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/kyc/:id',
+            },
+        ],
+        news: [
+            {
+                title: '新闻网',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/news/:id',
+            },
+        ],
+        rsc: [
+            {
+                title: '党委教师工作部、人事处',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/rsc/:id',
+            },
+        ],
+        skb: [
+            {
+                title: '马克思主义学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/skb/:id',
+            },
+        ],
+        slxy: [
+            {
+                title: '数理学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/slxy/:id',
+            },
+        ],
+        tyb: [
+            {
+                title: '体育学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/tyb/:id',
+            },
+        ],
+        wgyxy: [
+            {
+                title: '外国语学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/wgyxy/:id',
+            },
+        ],
+        xxgk: [
+            {
+                title: '信息公开网',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/xxgk/:id',
+            },
+        ],
+        yjsc: [
+            {
+                title: '研究生院/研工部',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/yjsc/:id',
+            },
+        ],
+        yjzx: [
+            {
+                title: '人文艺术学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/yjzx/:id',
+            },
+        ],
+        zdhxy: [
+            {
+                title: '自动化工程学院',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/zdhxy/:id',
+            },
+        ],
+        zs: [
+            {
+                title: '本科招生网',
+                docs: 'https://docs.rsshub.app/university.html#shang-hai-dian-li-da-xue',
+                source: ['/:id/list.htm'],
+                target: '/shiep/zs/:id',
+            },
+        ],
+    },
+};

package/lib/v2/shiep/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:type/:id?', require('./index'));
+};

package/lib/v2/shiep/templates/description.art

@@ -0,0 +1 @@
+{{@ description }}

package/lib/v2/shoppingdesign/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/posts': ['miles170'],
+};

package/lib/v2/shoppingdesign/posts.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const asyncPool = require('tiny-async-pool');
+
+module.exports = async (ctx) => {
+    // sn_f parameter is required to prevent redirection
+    const currentUrl = 'https://www.shoppingdesign.com.tw/post?sn_f=1';
+    const response = await got(currentUrl);
+    const $ = cheerio.load(response.data);
+    const items = [];
+    // maximum parallel requests on the target website are limited to 11.
+    for await (const data of asyncPool(10, $('article-item'), (item) => {
+        item = $(item);
+        const link = item.attr('url');
+        return ctx.cache.tryGet(link, async () => {
+            const response = await got(`${link}?sn_f=1`);
+            const $ = cheerio.load(response.data);
+            const article = $('.left article .htmlview');
+            article.find('d-image').each(function () {
+                $(this).replaceWith(`<img src="${$(this).attr('lg')}">`);
+            });
+
+            return {
+                title: $('.left article .top_info h1').text(),
+                author: $('meta[name="my:author"]').attr('content'),
+                description: article.html(),
+                category: $('meta[name="my:category"]').attr('content'),
+                pubDate: parseDate($('meta[name="my:publish"]').attr('content')),
+                link,
+            };
+        });
+    })) {
+        items.push(data);
+    }
+
+    ctx.state.data = {
+        title: $('meta[property="og:title"]').attr('content'),
+        link: currentUrl,
+        description: $('meta[property="og:description"]').attr('content'),
+        item: items,
+    };
+};

package/lib/v2/shoppingdesign/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'shoppingdesign.com.tw': {
+        _name: 'Shopping Design',
+        www: [
+            {
+                title: '文章列表',
+                docs: 'https://docs.rsshub.app/design.html#shopping-design',
+                source: '/post',
+                target: '/shoppingdesign/posts',
+            },
+        ],
+    },
+};

package/lib/v2/shoppingdesign/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/posts', require('./posts'));
+};

package/lib/v2/skysports/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:team': ['nczitzk'],
+};

package/lib/{routes → v2}/skysports/news.js

@@ -7,6 +7,7 @@ module.exports = async (ctx) => {
 
     const rootUrl = 'https://www.skysports.com';
     const currentUrl = `${rootUrl}/${team}-news`;
+
     const response = await got({
         method: 'get',
         url: currentUrl,
@@ -14,19 +15,19 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('.news-list__headline-link')
-        .map((_, item) => {
+    let items = $('.news-list__headline-link')
+        .toArray()
+        .map((item) => {
             item = $(item);
 
             return {
                 title: item.text(),
                 link: item.attr('href'),
             };
-        })
-        .get();
+        });
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',
@@ -35,10 +36,11 @@ module.exports = async (ctx) => {
 
                 const content = cheerio.load(detailResponse.data);
 
+                content('.roadblock').remove();
                 content('.sdc-article-widget, .sdc-site-layout-sticky-region').remove();
 
-                item.description = content('.sdc-article-body').html();
-                item.pubDate = parseDate(detailResponse.data.match(/"datePublished": "(.*)","dateModified"/)[1]);
+                item.description = content('.sdc-article-body, .polaris-tile-group-separator').html();
+                item.pubDate = parseDate(detailResponse.data.match(/"datePublished": ?"(.*)","dateModified"/)[1]);
 
                 return item;
             })

package/lib/v2/skysports/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'skysports.com': {
+        _name: 'Sky Sports',
+        '.': [
+            {
+                title: 'News',
+                docs: 'https://docs.rsshub.app/new-media.html#sky-sports-news',
+                source: ['/'],
+                target: (params, url) => `/skysports/news/${new URL(url).toString().match(/\/(.*)-news$/)[1]}`,
+            },
+        ],
+    },
+};

package/lib/v2/skysports/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/news/:team', require('./news'));
+};

package/lib/v2/smzdm/haowen_fenlei.js

@@ -24,18 +24,24 @@ module.exports = async (ctx) => {
             return {
                 title: item.find('h2.item-name a').text(),
                 link: item.find('h2.item-name a').attr('href'),
-                pubDate: timezone(parseDate(item.find('span.time').text())),
+                description: item.find('.item-info').html(),
+                author: item.find('.nickname').text(),
+                pubDate: timezone(parseDate(item.find('span.time').text(), ['HH:mm', 'MM-DD HH:mm', 'YYYY-MM-DD HH:mm']), 8),
             };
         });
 
     const out = await Promise.all(
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
-                const response = await got(item.link);
-                const $ = cheerio.load(response.data);
-                item.description = $('article').html();
-                item.pubDate = timezone(parseDate($('meta[property="og:release_date"]').attr('content')), 8);
-                item.author = $('meta[property="og:author"]').attr('content');
+                try {
+                    const response = await got(item.link);
+                    const $ = cheerio.load(response.data);
+                    item.description = $('article').html();
+                    item.pubDate = timezone(parseDate($('meta[property="og:release_date"]').attr('content')), 8);
+                    item.author = $('meta[property="og:author"]').attr('content');
+                } catch (err) {
+                    // 404
+                }
 
                 return item;
             })

package/lib/v2/solidot/main.js

@@ -6,9 +6,14 @@
 const got = require('@/utils/got'); // get web content
 const cheerio = require('cheerio'); // html parser
 const get_article = require('./_article');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const type = ctx.params.type ?? 'www';
+    if (!isValidHost(type)) {
+        throw Error('Invalid type');
+    }
+
     const base_url = `https://${type}.solidot.org`;
     const response = await got({
         method: 'get',

package/lib/v2/telegram/channel.js

@@ -120,6 +120,7 @@ module.exports = async (ctx) => {
             list
                 .map((index, item) => {
                     item = $(item);
+                    let extra = null;
 
                     /* message types */
                     let msgTypes = [];
@@ -208,6 +209,15 @@ module.exports = async (ctx) => {
                                 fwdFrom += ` (${fwdFromAuthorObj.text()})`;
                             }
                             fwdFrom += '</p>';
+
+                            extra = {
+                                links: [
+                                    {
+                                        type: 'repost',
+                                        url: userLink,
+                                    },
+                                ],
+                            };
                         }
                         return fwdFrom;
                     };
@@ -227,18 +237,26 @@ module.exports = async (ctx) => {
                             const replyTextObj = replyObj.find('.tgme_widget_message_text');
                             const replyText = replyTextObj.length ? `<p>${replyTextObj.html()}</p>` : '';
 
+                            extra = {
+                                links: [
+                                    {
+                                        type: 'reply',
+                                        url: replyLink,
+                                    },
+                                ],
+                            };
                             if (replyLink !== '') {
-                                return `<blockquote>
+                                return `<div class="rsshub-quote"><blockquote>
                                     <p><a href='${replyLink}'><b>${replyAuthor}</b>${viaBotText}:</a></p>
                                     ${replyMetaText}
                                     ${replyText}
-                                </blockquote>`;
+                                </blockquote></div>`;
                             } else {
-                                return `<blockquote>
+                                return `<div class="rsshub-quote"><blockquote>
                                     <p><b>${replyAuthor}</b>${viaBotText}:</p>
                                     ${replyMetaText}
                                     ${replyText}
-                                </blockquote>`;
+                                </blockquote></div>`;
                             }
                         } else {
                             return '';
@@ -576,6 +594,8 @@ module.exports = async (ctx) => {
                         enclosure_url: voiceUrl,
                         itunes_duration: voiceDuration(),
                         enclosure_type: 'audio/ogg',
+
+                        _extra: extra,
                     };
                 })
                 .get()

package/lib/v2/test/index.js

@@ -176,6 +176,52 @@ module.exports = async (ctx) => {
             pubDate: new Date(1546272000000).toUTCString(),
             author: `DIYgod0`,
         });
+    } else if (ctx.params.id === 'json') {
+        item.push(
+            {
+                title: 'Title0',
+                pubDate: new Date(`2019-3-1`).toUTCString(),
+                link: `https://github.com/DIYgod/RSSHub/issues/-3`,
+            },
+            {
+                title: 'Title1',
+                description: 'Description1',
+                pubDate: new Date(`2019-3-1`).toUTCString(),
+                link: `https://github.com/DIYgod/RSSHub/issues/-2`,
+                author: `DIYgod0 `,
+                category: 'Category0',
+            },
+            {
+                title: 'Title2 HTML in description',
+                description: '<a href="https://github.com/DIYgod/RSSHub">RSSHub</a>',
+                pubDate: new Date(`2019-3-1`).toUTCString(),
+                updated: new Date(`2019-3-2`).toUTCString(),
+                link: `https://github.com/DIYgod/RSSHub/issues/-1`,
+                author: [{ name: ' DIYgod1' }, { name: 'DIYgod2 ' }],
+                category: ['Category0', 'Category1'],
+            },
+            {
+                title: 'Title3 HTML in content',
+                content: {
+                    html: '<a href="https://github.com/DIYgod/RSSHub">DIYgod/RSSHub</a>',
+                },
+                pubDate: new Date(`2019-3-1`).toUTCString(),
+                updated: new Date(`2019-3-2`).toUTCString(),
+                link: `https://github.com/DIYgod/RSSHub/issues/0`,
+                author: [{ name: '   DIYgod3' }, { name: 'DIYgod4   ' }, { name: 'DIYgod5   ' }],
+                category: ['Category1'],
+                enclosure_url: 'https://github.com/DIYgod/RSSHub/issues/0',
+                enclosure_type: 'image/jpeg',
+                enclosure_length: 3661,
+                itunes_duration: 36610,
+            },
+            {
+                title: 'Title4 author is null',
+                pubDate: new Date(`2019-3-1`).toUTCString(),
+                link: `https://github.com/DIYgod/RSSHub/pull/11555`,
+                author: null,
+            }
+        );
     }
 
     for (let i = 1; i < 6; i++) {

package/lib/v2/thepaper/channel.js

@@ -5,16 +5,22 @@ const got = require('@/utils/got');
 module.exports = async (ctx) => {
     const { id } = ctx.params;
     const channel_url = `https://m.thepaper.cn/channel/${id}`;
-    const response = await got(channel_url);
-    const data = JSON.parse(cheerio.load(response.data)('#__NEXT_DATA__').html());
-    const list = data.props.pageProps.data.list;
+    const channel_url_resp = await got(channel_url);
+    const channel_url_data = JSON.parse(cheerio.load(channel_url_resp.data)('#__NEXT_DATA__').html());
+
+    const resp = await got.post('https://api.thepaper.cn/contentapi/nodeCont/getByChannelId', {
+        json: {
+            channelId: id,
+        },
+    });
+    const list = resp.data.data.list;
 
     const items = await Promise.all(list.map((item) => utils.ProcessItem(item, ctx)));
     ctx.state.data = {
-        title: `澎湃新闻频道 - ${utils.ChannelIdToName(id, data)}`,
+        title: `澎湃新闻频道 - ${utils.ChannelIdToName(id, channel_url_data)}`,
         link: channel_url,
         item: items,
         itunes_author: '澎湃新闻',
-        image: utils.ExtractLogo(response),
+        image: utils.ExtractLogo(channel_url_resp),
     };
 };

package/lib/v2/thepaper/list.js

@@ -5,17 +5,23 @@ const got = require('@/utils/got');
 module.exports = async (ctx) => {
     const { id } = ctx.params;
     const list_url = `https://m.thepaper.cn/list/${id}`;
-    const response = await got(list_url);
-    const data = JSON.parse(cheerio.load(response.data)('#__NEXT_DATA__').html());
-    const pagePropsData = data.props.pageProps.data;
+    const list_url_resp = await got(list_url);
+    const list_url_data = JSON.parse(cheerio.load(list_url_resp.data)('#__NEXT_DATA__').html());
+
+    const resp = await got.post('https://api.thepaper.cn/contentapi/nodeCont/getByNodeIdPortal', {
+        json: {
+            nodeId: id,
+        },
+    });
+    const pagePropsData = resp.data.data;
     const list = pagePropsData.list;
 
     const items = await Promise.all(list.map((item) => utils.ProcessItem(item, ctx)));
     ctx.state.data = {
-        title: `澎湃新闻栏目 - ${utils.ListIdToName(id, data)}`,
+        title: `澎湃新闻栏目 - ${utils.ListIdToName(id, list_url_data)}`,
         link: list_url,
         item: items,
         itunes_author: '澎湃新闻',
-        image: pagePropsData.nodeInfo?.pic ?? utils.ExtractLogo(response),
+        image: pagePropsData.nodeInfo?.pic ?? utils.ExtractLogo(list_url_resp),
     };
 };

package/lib/v2/thepaper/utils.js

@@ -2,14 +2,13 @@ const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const got = require('@/utils/got');
 const { art } = require('@/utils/render');
-const timezone = require('@/utils/timezone');
 const path = require('path');
 
 const defaultRssItem = (item) => ({
     title: item.name,
     link: item.link,
     description: item.name,
-    pubDate: timezone(parseDate(item.pubTimeLong), +8),
+    pubDate: parseDate(item.pubTimeLong),
     media: {
         content: {
             url: item.pic,
@@ -52,7 +51,7 @@ module.exports = {
                         link: itemUrl,
                         description,
                         category: contentDetail.tagList?.map((t) => t.tag) ?? [],
-                        pubDate: timezone(parseDate(item.pubTimeLong || contentDetail.pubTime || contentDetail.publishTime), +8),
+                        pubDate: parseDate(item.pubTimeLong || contentDetail.pubTime || contentDetail.publishTime),
                         author: contentDetail.author || '',
                         media: {
                             content: {

package/lib/v2/twitter/utils.js

@@ -205,6 +205,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
             if (quoteData) {
                 quoteData.full_text = quoteData.full_text || quoteData.text;
                 const author = quoteData.user;
+                quote += '<div class="rsshub-quote">';
                 if (readable) {
                     quote += `<br clear='both' /><div style='clear: both'></div>`;
                     quote += `<blockquote style='background: #80808010;border-top:1px solid #80808030;border-bottom:1px solid #80808030;margin:0;padding:5px 20px;'>`;
@@ -260,6 +261,7 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
                 if (readable) {
                     quote += `</blockquote>`;
                 }
+                quote += '</div>';
             } else {
                 url = formatUrl(item);
             }
@@ -382,6 +384,32 @@ const ProcessFeed = (ctx, { data = [] }, params = {}) => {
             description,
             pubDate: parseDate(item.created_at),
             link: `https://twitter.com/${item.user.screen_name}/status/${item.id_str}`,
+
+            _extra:
+                (isRetweet && {
+                    links: [
+                        {
+                            type: 'repost',
+                        },
+                    ],
+                }) ||
+                (item.is_quote_status && {
+                    links: [
+                        {
+                            url: `https://twitter.com/${item.quoted_status?.user?.screen_name}/status/${item.quoted_status?.id_str}`,
+                            type: 'quote',
+                        },
+                    ],
+                }) ||
+                (item.in_reply_to_screen_name &&
+                    item.in_reply_to_status_id_str && {
+                        links: [
+                            {
+                                url: `https://twitter.com/${item.in_reply_to_screen_name}/status/${item.in_reply_to_status_id_str}`,
+                                type: 'reply',
+                            },
+                        ],
+                    }),
         };
     });
 };

package/lib/v2/typora/changelog-dev.js

@@ -0,0 +1,31 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const currentUrl = 'https://typora.io/releases/dev';
+    const response = await got(currentUrl);
+
+    const $ = cheerio.load(response.data);
+
+    const items = $('h2')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                link: `${currentUrl}#${item.text()}`,
+                description: item
+                    .nextUntil('h2')
+                    .toArray()
+                    .map((item) => $(item).html())
+                    .join(''),
+            };
+        });
+
+    ctx.state.data = {
+        title: `Typora Changelog - Dev`,
+        link: currentUrl,
+        description: 'Typora Changelog',
+        item: items,
+    };
+};