rsshub 1.0.0-master.fb68956 → 1.0.0-master.fbabfc2

package/lib/v2/gq/tw/index.js

@@ -0,0 +1,165 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const md = require('markdown-it')({
+    linkify: true,
+});
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const baseUrl = 'https://www.gq.com.tw';
+const parsePreloadedStateJSON = ($) =>
+    JSON.parse(
+        $('script[type="text/javascript"]')
+            .text()
+            .match(/window\.__PRELOADED_STATE__ = ({.*?});/)[1]
+    );
+const largestImage = (sources, id) => {
+    if (!id) {
+        let maxWidth = 0;
+        let maxUrl = '';
+        Object.values(sources).forEach((size) => {
+            if (size.width > maxWidth) {
+                maxWidth = size.width;
+                maxUrl = size.url;
+            }
+        });
+        return maxUrl;
+    }
+    const url = sources[Object.keys(sources)[0]].url;
+    const filename = url.substring(url.lastIndexOf('/') + 1);
+    return `https://media.gq.com.tw/photos/${id}/${filename}`;
+};
+
+module.exports = async (ctx) => {
+    const { caty, subcaty } = ctx.params;
+    const link = `${baseUrl}${caty ? `/${caty}` : ''}${subcaty ? `/${subcaty}` : ''}`;
+    const { data: response } = await got(link);
+    const $ = cheerio.load(response);
+    const { transformed } = parsePreloadedStateJSON($);
+
+    const list = transformed.bundle.containers
+        .filter((item) => item.items)
+        .map((item) =>
+            item.items.map((item) => ({
+                title: item.source?.hed || item.dangerousHed,
+                description: item.source?.dek || item.dangerousDek,
+                link: (item.url.startsWith('http') ? item.url : `${baseUrl}${item.url}`).split('#intcid')[0],
+                pubDate: parseDate(item.pubDate),
+                author: item.contributors.author.items.map((item) => item.name).join(', '),
+            }))
+        )
+        .flat();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+                const data = JSON.parse($('script[type="application/ld+json"]').first().text());
+                const { transformed } = parsePreloadedStateJSON($);
+                const VideoObject = data['@type'] === 'VideoObject' ? true : false;
+                const lede = !VideoObject ? transformed.article.headerProps.lede : null;
+
+                let articleBody = '';
+                if (!VideoObject) {
+                    // typical article
+                    // articleBody inludes non standard markdown syntax
+                    articleBody = cheerio.load(md.render(data.articleBody.replace(/\{: target="_blank"\}/g, '')).replace(/\{: #link\}/g, ''), null, false);
+                    articleBody('a').each((_, item) => {
+                        item = $(item);
+                        if (item.text().startsWith('#video:')) {
+                            if (item.text().match(/youtube/)) {
+                                const videoId = item.text().match(/.*https:\/\/www\.youtube\.com\/embed\/(.*)\s/)[1];
+                                item.replaceWith(art(path.join(__dirname, '../templates/youtube.art'), { videoId }));
+                            }
+                        }
+                        if (item.text().startsWith('![#image:')) {
+                            const imageId = item.text().match(/!\[#image: \/photos\/(.*?)\]/)[1];
+                            const imgInfo = transformed.article.lightboxImages.find((item) => item.id === imageId);
+                            item.html(
+                                art(path.join(__dirname, '../templates/img.art'), {
+                                    src: largestImage(imgInfo.sources, imageId),
+                                    alt: imgInfo.dangerousCaption,
+                                })
+                            );
+                        }
+                    });
+                    articleBody('p').each((_, item) => {
+                        item = $(item);
+                        if (item.text().startsWith('+++')) {
+                            item.remove();
+                        }
+                        if (item.text().startsWith('[#image:')) {
+                            const imageId = item.text().match(/\[#image: \/photos\/(.*?)\]/)[1];
+                            const imgInfo = transformed.article.lightboxImages.find((item) => item.id === imageId);
+                            item.replaceWith(
+                                art(path.join(__dirname, '../templates/img.art'), {
+                                    src: largestImage(imgInfo.sources, imageId),
+                                    alt: imgInfo.dangerousCaption,
+                                })
+                            );
+                        }
+                        if (item.text().startsWith('[#article:')) {
+                            const articleId = item.text().match(/\[#article: \/articles\/(.*?)\]/)[1];
+                            const articleProps = transformed.article.body.filter((i) => i[0] === 'inline-embed').find((i) => i[1].ref === articleId)[1].props;
+                            item.replaceWith(
+                                art(path.join(__dirname, '../templates/embed-article.art'), {
+                                    url: `${baseUrl}${articleProps.url}`,
+                                    text: articleProps.dangerousHed,
+                                })
+                            );
+                        }
+                        if (item.text().startsWith('[#product:')) {
+                            const productId = item.text().match(/\[#product: \/products\/(.*?)\]/)[1];
+                            const productProps = transformed.article.body.filter((i) => i[0] === 'inline-embed').find((i) => i[1].ref === productId)[1].props;
+                            item.replaceWith(
+                                art(path.join(__dirname, '../templates/embed-product.art'), {
+                                    img: art(path.join(__dirname, '../templates/img.art'), {
+                                        src: largestImage(productProps.image.sources),
+                                    }),
+                                    productProps,
+                                })
+                            );
+                        }
+                        if (item.text().startsWith('[#instagram:')) {
+                            const instagramHref = item.text().match(/\[#instagram: (.*?)\]/)[1];
+                            item.replaceWith(
+                                art(path.join(__dirname, '../templates/embed-article.art'), {
+                                    url: instagramHref,
+                                    text: instagramHref,
+                                })
+                            );
+                        }
+                    });
+                    item.description = art(path.join(__dirname, '../templates/tw.art'), {
+                        dangerousDek: transformed.article.headerProps.dangerousDek, // quote
+                        lede: art(path.join(__dirname, '../templates/img.art'), {
+                            // header image
+                            src: lede?.sources ? largestImage(lede.sources, lede.id) : null,
+                            alt: lede?.caption,
+                        }),
+                        articleBody: articleBody.html(),
+                    });
+                } else {
+                    // is VideoObject
+                    item.description = art(path.join(__dirname, '../templates/videoObject.art'), {
+                        poster: transformed.video.metaImageUrl,
+                        sources: transformed.video.sources,
+                        articleBody: data.description,
+                    });
+                }
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${transformed.coreDataLayer.content.contentTitle} | ${transformed.coreDataLayer.content.brand}`,
+        link,
+        image: `${baseUrl}${transformed.logo.sources.sm.url}`,
+        item: items,
+        allowEmpty: true,
+    };
+};

package/lib/v2/guandian/index.js

@@ -0,0 +1,37 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const baseUrl = 'https://www.guandian.cn';
+
+module.exports = async (ctx) => {
+    const cat = ctx.params.category;
+    const reqUrl = `${baseUrl}/${cat}/`;
+    const response = await got(reqUrl);
+    const $ = cheerio.load(response.data);
+
+    const items = $('.con_l li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('.con_l_con_r a');
+            const link = a.attr('href');
+            const title = a.find('h3').text();
+            const desc = item.find('.con_l_con_r p').text();
+
+            const dateStr = item.find('#keyword span').text();
+            return {
+                title,
+                link,
+                description: desc,
+                pubDate: timezone(parseDate(dateStr), 8),
+            };
+        });
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: reqUrl,
+        item: items,
+    };
+};

package/lib/v2/guandian/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category': ['drgnchan'],
+};

package/lib/v2/guandian/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'guandian.cn': {
+        _name: '观点网',
+        www: [
+            {
+                title: '资讯',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-dian-wang-zi-xun',
+                source: ['/:category'],
+                target: '/guandian/:category',
+            },
+        ],
+    },
+};

package/lib/v2/guandian/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:category', require('./index'));
+};

package/lib/v2/gumroad/index.js

@@ -2,10 +2,14 @@ const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { art } = require('@/utils/render');
 const path = require('path');
+const { isValidHost } = require('@/utils/valid-host');
 
 module.exports = async (ctx) => {
     const username = ctx.params.username;
     const products = ctx.params.products;
+    if (!isValidHost(username)) {
+        throw Error('Invalid username');
+    }
     const url = `https://${username}.gumroad.com/l/${products}`;
 
     const response = await got(url);

package/lib/v2/hackmd/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/profile/:path': ['Yukaii', 'kaiix'],
+};

package/lib/v2/hackmd/profile.js

@@ -0,0 +1,24 @@
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    const path = ctx.params.path;
+
+    const response = await got({
+        method: 'get',
+        url: `https://hackmd.io/api/@${path}/overview`,
+    });
+    const data = response.data;
+
+    const profile = data.user || data.team;
+    ctx.state.data = {
+        title: `${profile.name}'s Profile`,
+        link: `https://hackmd.io/@${path}`,
+        description: `${profile.name}'s profile on HackMD`,
+        item: data.notes.map((note) => ({
+            title: note.title,
+            description: `<pre>${note.content}</pre>`,
+            pubDate: new Date(note.publishedAt).toUTCString(),
+            link: `https://hackmd.io/@${path}/${note.permalink || note.shortId}`,
+        })),
+    };
+};

package/lib/v2/hackmd/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'hackmd.io': {
+        _name: 'HackMD',
+        '.': [
+            {
+                title: 'Profile',
+                docs: 'http://docs.rsshub.app/programming.html#hackmd',
+                source: ['/:profile'],
+                target: (params) => `/hackmd/profile/${params.replace('@', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/hackmd/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/profile/:path', require('./profile'));
+};

package/lib/v2/hellogithub/maintainer.js

@@ -5,5 +5,5 @@ module.exports = {
     '/month': ['moke8', 'nczitzk'],
     '/ranking/:type?': ['moke8', 'nczitzk'],
     '/report/:type?': ['nczitzk'],
-    '/volume': ['nczitzk'],
+    '/volume': ['nczitzk', 'CaoMeiYouRen'],
 };

package/lib/v2/hellogithub/volume.js

@@ -4,6 +4,7 @@ const path = require('path');
 const md = require('markdown-it')({
     html: true,
 });
+const cheerio = require('cheerio');
 
 art.defaults.imports.render = function (string) {
     return md.render(string);
@@ -11,28 +12,28 @@ art.defaults.imports.render = function (string) {
 
 module.exports = async (ctx) => {
     const rootUrl = 'https://hellogithub.com';
-    const currentUrl = `${rootUrl}/periodical/volume/`;
+    const apiUrl = 'https://api.hellogithub.com/v1/periodical/';
 
-    const buildResponse = await got({
+    const periodicalResponse = await got({
         method: 'get',
-        url: rootUrl,
+        url: apiUrl,
     });
-
-    const buildId = buildResponse.data.match(/"buildId":"(.*?)",/)[1];
-
-    const apiUrl = `${rootUrl}/_next/data/${buildId}/periodical/volume.json`;
-
-    const response = await got({
+    const current = periodicalResponse.data.volumes[0].num;
+    const currentUrl = `${rootUrl}/periodical/volume/${current}`;
+    const buildResponse = await got({
         method: 'get',
-        url: apiUrl,
+        url: currentUrl,
     });
 
-    const data = response.data;
+    const $ = cheerio.load(buildResponse.data);
+
+    const text = $('#__NEXT_DATA__').text();
+    const response = JSON.parse(text);
+    const data = response.props;
     const id = data.pageProps.volume.current_num;
 
     const items = [
         {
-            guid: id,
             title: `No.${id}`,
             link: `${rootUrl}/periodical/volume/${id}`,
             description: art(path.join(__dirname, 'templates/volume.art'), {

package/lib/v2/hljucm/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/yjsy/:category?': ['nczitzk'],
+};

package/lib/v2/hljucm/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'hljucm.net': {
+        _name: '黑龙江中医药大学',
+        yjsy: [
+            {
+                title: '研究生院',
+                docs: 'https://docs.rsshub.app/university.html#hei-long-jiang-zhong-yi-yao-da-xue',
+                source: ['/index/:category', '/index'],
+                target: (params) => `/hljucm/yjsy/${params.category.replace('.htm', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/hljucm/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/yjsy/:category?', require('./yjsy'));
+};

package/lib/v2/hljucm/yjsy.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'xwdt';
+
+    const rootUrl = 'https://yjsy.hljucm.net';
+    const currentUrl = `${rootUrl}/index/${category}.htm`;
+
+    const response = await got(currentUrl);
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('.postlist a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.attr('title'),
+                link: new URL(item.attr('href'), currentUrl).href,
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('#vsb_newscontent').html();
+                item.pubDate = timezone(parseDate(content('.timestyle56043').text()), +8);
+
+                const files = detailResponse.data.match(/<span>附件【<a href="(.*)"><span>(.*)<\/span><\/a>】<\/span>/g);
+
+                if (files) {
+                    for (const file of files) {
+                        const { link, name } = file.match(/【<a href="(?<link>.*)"><span>(?<name>.*)<\/span><\/a>】/).groups;
+                        item.description += `<a href="${link}">${name}</a>`;
+                    }
+                }
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/hotukdeals/hottest.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const { JSDOM } = require('jsdom');
+
+module.exports = async (ctx) => {
+    const data = await got.get(`https://www.hotukdeals.com/`, {
+        headers: {
+            Referer: `https://www.hotukdeals.com/`,
+        },
+    });
+
+    const dom = new JSDOM(data.data, {
+        runScripts: 'dangerously',
+    });
+    const threads = dom.window.__INITIAL_STATE__.widgets.hottestWidget.threads;
+
+    ctx.state.data = {
+        title: `hotukdeals hottest`,
+        link: `https://www.hotukdeals.com/`,
+        item: threads.map((item) => ({
+            title: item.title,
+            description: `<img src="https://images.hotukdeals.com/${item.mainImage.path}/${item.mainImage.name}/re/768x768/qt/60/${item.mainImage.name}.jpg"><br>${item.temperature}° ${item.title}<br>${item.displayPrice}`,
+            link: item.url,
+        })),
+    };
+};

package/lib/v2/hotukdeals/maintainer.js

@@ -1,3 +1,4 @@
 module.exports = {
     '/:type': ['DIYgod'],
+    '/hottest': ['DIYgod'],
 };

package/lib/v2/hotukdeals/radar.js

@@ -26,6 +26,12 @@ module.exports = {
                 source: ['/'],
                 target: '/hotukdeals/discussed',
             },
+            {
+                title: 'hottest',
+                docs: 'https://docs.rsshub.app/shopping.html#hotukdeals',
+                source: ['/'],
+                target: '/hotukdeals/hottest',
+            },
         ],
     },
 };

package/lib/v2/hotukdeals/router.js

@@ -1,3 +1,4 @@
 module.exports = function (router) {
+    router.get('/hottest', require('./hottest'));
     router.get('/:type', require('./index'));
 };

package/lib/v2/huanqiu/index.js

@@ -1,6 +1,7 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const { isValidHost } = require('@/utils/valid-host');
 
 function getKeysRecursive(dic, key, attr, array) {
     Object.values(dic).forEach((v) => {
@@ -15,6 +16,10 @@ function getKeysRecursive(dic, key, attr, array) {
 
 module.exports = async (ctx) => {
     const category = ctx.params.category ?? 'china';
+    if (!isValidHost(category)) {
+        throw Error('Invalid category');
+    }
+
     const host = 'https://' + category + '.huanqiu.com';
 
     const resp = await got({

package/lib/v2/huxiu/article.js

@@ -1,23 +1,26 @@
 const got = require('@/utils/got');
-const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
-    const link = 'https://www.huxiu.com/article';
-    const response = await got({
-        method: 'get',
-        url: link,
+    const link = `${utils.baseUrl}/article/`;
+    const { data } = await got.post(`${utils.articleApi}/web/article/articleList`, {
         headers: {
             Referer: link,
         },
+        form: {
+            platform: 'www',
+            pagesize: ctx.query.limit ? parseInt(ctx.query.limit) : 22,
+        },
     });
 
-    const $ = cheerio.load(response.data);
-
-    const list = $('.article-item-wrap > a')
-        .slice(0, 10)
-        .get()
-        .map((e) => $(e).attr('href'));
+    const list = data.data.dataList.map((item) => ({
+        title: item.title,
+        link: `${utils.baseUrl}/article/${item.aid}.html`,
+        description: item.summary,
+        pubDate: parseDate(item.dateline, 'X'),
+        author: item.user_info.username,
+    }));
 
     const items = await utils.ProcessFeed(list, ctx.cache);
 

package/lib/v2/huxiu/author.js

@@ -1,28 +1,14 @@
 const got = require('@/utils/got');
-const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const link = `https://www.huxiu.com/member/${id}/article.html`;
-
-    const response = await got({
-        method: 'get',
-        url: link,
-        headers: {
-            Referer: link,
-        },
-    });
-
-    const $ = cheerio.load(response.data);
-    const author = $('.author-name').text().trim();
+    const link = `${utils.baseUrl}/member/${id}/article.html`;
 
     const articleList = await got({
         method: 'post',
-        url: 'https://account-api.huxiu.com/web/article/articleList',
-        headers: {
-            Referer: 'https://www.huxiu.com/',
-        },
+        url: `${utils.accountApi}/web/article/articleList`,
         form: {
             platform: 'www',
             uid: id,
@@ -31,15 +17,20 @@ module.exports = async (ctx) => {
         },
     }).json();
 
-    const list = articleList.data.datalist.slice(0, 10).map((item) => item.url);
+    const list = articleList.data.datalist.map((item) => ({
+        title: item.title,
+        link: `${utils.baseUrl}/article/${item.aid}.html`,
+        description: item.summary,
+        pubDate: parseDate(item.time),
+    }));
 
     const items = await utils.ProcessFeed(list, ctx.cache);
 
-    const authorInfo = `虎嗅网 - ${author}`;
+    const authorInfo = `虎嗅网 - ${items[0].author}`;
+
     ctx.state.data = {
         title: authorInfo,
         link,
-        description: authorInfo,
         item: items,
     };
 };