rsshub 1.0.0-master.fe69819 → 1.0.0-master.fe775f5

package/lib/v2/eprice/rss.js

@@ -30,9 +30,10 @@ module.exports = async (ctx) => {
                 // remove unwanted elements
                 $('noscript').remove();
                 $('div[id^=dablewidget]').remove();
+                $('div[class^=parallax-ads]').remove();
                 $('.adsbygoogle, .join-eprice-fb, .teads').remove();
                 $('div.ad-336x280-g, div.ad-728x90-g').remove();
-                $('div.clear, div.news-vote, div.text-left, div.signature').remove();
+                $('div.clear, div.news-vote, div.signature').remove();
                 $('ul.inner, ul.navigator, ul.infobar').remove();
                 $('iframe[src^="https://www.facebook.com/plugins/like.php"]').remove();
 
@@ -76,7 +77,8 @@ module.exports = async (ctx) => {
                 delete item.isoDate;
 
                 item.description = art(path.join(__dirname, 'templates/description.art'), {
-                    desc: $('div.user-comment-block').html() ?? $('div.content').html() ?? $('li.inner').html(),
+                    // tw || tw || hk || hk
+                    desc: $('div.user-comment-block').html() || $('div.content').html() || $('li.inner').html() || $('div.section-content').html(),
                 });
                 item.pubDate = parseDate(item.pubDate);
 

package/lib/v2/ff14/ff14_global.js

@@ -9,7 +9,7 @@ module.exports = async (ctx) => {
 
     const response = await got({
         method: 'get',
-        url: `http://${lang}.lodestonenews.com/news/${type}`,
+        url: `https://lodestonenews.com/news/${type}?locale=${lang}`,
     });
 
     let data;

package/lib/v2/flyert/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/preferential': ['howel52'],
+    '/creditcard/:bank': ['nicolaszf'],
+};

package/lib/v2/flyert/preferential.js

@@ -0,0 +1,48 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const iconv = require('iconv-lite');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const gbk2utf8 = (s) => iconv.decode(s, 'gbk');
+const host = 'https://www.flyert.com';
+const target = `${host}/forum.php?mod=forumdisplay&sum=all&fid=all&catid=322`;
+
+module.exports = async (ctx) => {
+    const response = await got(target, {
+        responseType: 'buffer',
+    });
+
+    const $ = cheerio.load(gbk2utf8(response.data));
+    const list = $('.comiis_wzli')
+        .map((_index, item) => ({
+            title: $(item).find('.wzbt').text(),
+            link: `${host}/${$(item).find('.wzbt a').attr('href')}`,
+            description: $(item).find('.wznr > div:first-child').text(),
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link, {
+                    responseType: 'buffer',
+                });
+                const content = cheerio.load(gbk2utf8(detailResponse.data));
+
+                // remove top ad
+                content('div.artical_top').remove();
+
+                item.description = content('#artMain').html();
+                item.pubDate = timezone(parseDate(content('p.xg1 > span:nth-child(1)').attr('title') || content('p.xg1').text().split('|')[0], 'YYYY-M-D HH:mm'), +8);
+                return item;
+            })
+        )
+    );
+    ctx.state.data = {
+        title: '飞客茶馆优惠',
+        link: 'https://www.flyert.com/',
+        description: '飞客茶馆优惠',
+        item: items,
+    };
+};

package/lib/v2/flyert/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'flyert.com': {
+        _name: '飞客茶馆',
+        '.': [
+            {
+                title: '优惠信息',
+                docs: 'https://docs.rsshub.app/travel.html#fei-ke-cha-guan-you-hui-xin-xi',
+                source: '/',
+                target: '/flyert/preferential',
+            },
+            {
+                title: '信用卡',
+                docs: 'https://docs.rsshub.app/travel.html#fei-ke-cha-guan-xin-yong-ka',
+                source: '/',
+                target: '/flyert/creditcard/:bank',
+            },
+        ],
+    },
+};

package/lib/v2/flyert/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/preferential', require('./preferential'));
+    router.get('/creditcard/:bank', require('./creditcard'));
+};

package/lib/{routes → v2}/flyert/utils.js

@@ -13,7 +13,8 @@ async function load(link) {
 
     // 去除全文末尾多与内容
     $('.lookMore').remove();
-    $('script').remove();
+    $('script, style').remove();
+    $('#loginDialog').remove();
 
     // 修改图片中的链接
     $('ignore_js_op img').each(function () {

package/lib/v2/foresightnews/index.js

@@ -0,0 +1,88 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+const categories = {
+    '': {
+        id: 'feed',
+        title: '首页',
+    },
+    article: {
+        id: 'feed',
+        title: '文章',
+    },
+    news: {
+        id: 'news',
+        title: '快讯',
+    },
+    column: {
+        id: 'articles',
+        title: '专栏',
+    },
+};
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? '';
+    const id = ctx.params.id ?? '1';
+
+    const rootUrl = 'https://foresightnews.pro';
+    const currentUrl = `${rootUrl}/v1/${categories[category].id}?page=1&size=${ctx.query.limit ?? 30}${category === 'column' ? `&column_id=${id}` : ''}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    let items = response.data.data.list.map((item) =>
+        item.source_type
+            ? {
+                  type: item.source_type,
+                  title: item[item.source_type].title,
+                  author: item[item.source_type]?.author?.username ?? undefined,
+                  link: `${rootUrl}/${item.source_type}/detail/${item.source_id}`,
+                  category: item[item.source_type].tags ? item[item.source_type].tags.map((tag) => tag.name) : undefined,
+                  description: item[item.source_type]?.content ?? '',
+                  pubDate: parseDate(item[item.source_type].publishDate),
+              }
+            : {
+                  type: category,
+                  title: item.title,
+                  author: item?.author?.username ?? undefined,
+                  link: `${rootUrl}/${category === 'article' ? 'article' : 'news'}/detail/${item.id}`,
+                  category: item.tags ? item.tags.map((tag) => tag.name) : undefined,
+                  description: item.content ?? '',
+                  pubDate: parseDate(item.publishDate),
+              }
+    );
+
+    if (category === 'article') {
+        items = items.filter((item) => item.type === 'article');
+    }
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (item.type !== 'news') {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+
+                    const content = cheerio.load(detailResponse.data);
+
+                    item.description = content('.detail-body').html();
+                }
+
+                delete item.type;
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${categories[category].title} - Foresight News`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/foresightnews/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/': ['nczitzk'],
+    '/article': ['nczitzk'],
+    '/column/:id': ['nczitzk'],
+    '/news': ['nczitzk'],
+};

package/lib/v2/foresightnews/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'foresightnews.pro': {
+        _name: 'Foresight News',
+        '.': [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/new-media.html#foresight-news-shou-ye',
+                source: ['/article', '/'],
+                target: '/foresightnews',
+            },
+            {
+                title: '文章',
+                docs: 'https://docs.rsshub.app/new-media.html#foresight-news-wen-zhang',
+                source: ['/article', '/'],
+                target: '/foresightnews/article',
+            },
+            {
+                title: '快讯',
+                docs: 'https://docs.rsshub.app/new-media.html#foresight-news-kuai-xun',
+                source: ['/news', '/'],
+                target: '/foresightnews/news',
+            },
+            {
+                title: '专栏',
+                docs: 'https://docs.rsshub.app/new-media.html#foresight-news-zhuan-lan',
+                source: ['/column/detail/:id', '/'],
+                target: '/foresightnews/column/:id',
+            },
+        ],
+    },
+};

package/lib/v2/foresightnews/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:category?/:id?', require('./index'));
+};

package/lib/v2/gcores/category.js

@@ -0,0 +1,126 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+    const url = `https://www.gcores.com/${category}`;
+    const res = await got({
+        method: 'get',
+        url,
+    });
+    const data = res.data;
+    const $ = cheerio.load(data);
+    const feedTitle = $('title').text();
+
+    const list = $('.original.am_card.original-normal')
+        .map(function () {
+            const item = {
+                url: $(this).find('a.am_card_content.original_content').attr('href'),
+                title: $(this).find('h3.am_card_title').text(),
+                category: $(this).find('span.original_category>a').text(),
+            };
+            return item;
+        })
+        .get();
+
+    const out = await Promise.all(
+        list.map((item) => {
+            const articleUrl = `https://www.gcores.com${item.url}`;
+
+            return ctx.cache.tryGet(articleUrl, async () => {
+                const itemRes = await got({
+                    method: 'get',
+                    url: articleUrl,
+                });
+
+                const itemPage = itemRes.data;
+                const $ = cheerio.load(itemPage);
+
+                let articleData = await got(`https://www.gcores.com/gapi/v1${item.url}?include=media`);
+
+                articleData = articleData.data.data;
+                let cover;
+                if (articleData.attributes.cover) {
+                    cover = `<img src="https://image.gcores.com/${articleData.attributes.cover}" />`;
+                } else if (articleData.attributes.thumb) {
+                    cover = `<img src="https://image.gcores.com/${articleData.attributes.thumb}" />`;
+                } else {
+                    cover = '';
+                }
+
+                // replace figure with img
+                const articleContent = JSON.parse(articleData.attributes.content);
+                const entityRangeMap = {};
+                for (const block of articleContent.blocks || []) {
+                    if (block.entityRanges.length) {
+                        entityRangeMap[block.key] = block.entityRanges;
+                    }
+                }
+
+                $('figure').each((i, elem) => {
+                    const keyAttr = elem.attribs['data-offset-key'];
+                    const keyMatch = /^(\w+)-(\d+)-(\d)$/.exec(keyAttr);
+                    let actualContent = '';
+                    if (keyMatch) {
+                        const [, key, index] = keyMatch;
+                        if (entityRangeMap[key] && entityRangeMap[key][index]) {
+                            const entityKey = entityRangeMap[key] && entityRangeMap[key][index].key;
+                            const entity = articleContent.entityMap[entityKey];
+                            actualContent = convertEntityToContent(entity);
+                        }
+                    }
+
+                    if (actualContent) {
+                        $(elem).replaceWith(actualContent);
+                    }
+                });
+
+                // remove editor toolbar img
+                $('.md-editor-toolbar').replaceWith('');
+                // remove hidden tip block
+                $('.story_hidden').replaceWith('');
+
+                const content = $('.story.story-show').html();
+                return {
+                    title: item.title,
+                    description: cover + content,
+                    category: item.category,
+                    link: articleUrl,
+                    guid: articleUrl,
+                };
+            });
+        })
+    );
+    ctx.state.data = {
+        title: feedTitle,
+        link: url,
+        item: out,
+    };
+};
+
+function convertEntityToContent(entity) {
+    const { type, data } = entity;
+    switch (type) {
+        case 'IMAGE':
+            return `
+<figure>
+<img src="https://image.gcores.com/${data.path}" alt="${data.caption || ''}">
+${data.caption ? `<figcaption>${data.caption}</figcaption>` : ''}
+</figure>`;
+
+        case 'GALLERY':
+            return data.images
+                .map(
+                    (image, i, arr) => `
+<figure>
+<img src="https://image.gcores.com/${image.path}" alt="${image.caption || ''}">
+<figcaption>${data.caption || ''} (${i + 1}/${arr.length}) ${image.caption || ''}</figcaption>
+</figure>
+            `
+                )
+                .join('');
+
+        default:
+            return '';
+    }
+}

package/lib/v2/gcores/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/category/:category': ['MoguCloud', 'StevenRCE0'],
+    '/tag/:tag/:category?': ['StevenRCE0'],
+};

package/lib/v2/gcores/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'gcores.com': {
+        _name: '机核网',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/new-media.html#ji-he-wang-fen-lei',
+                source: ['/:category'],
+                target: '/gcores/category/:category',
+            },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/new-media.html#ji-he-wang-biao-qian',
+                source: ['/categories/:tag', '/'],
+                target: '/gcores/tag/:tag',
+            },
+        ],
+    },
+};

package/lib/v2/gcores/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/category/:category', require('./category'));
+    router.get('/tag/:tag/:category?', require('./tag'));
+};

package/lib/v2/gcores/tag.js

@@ -0,0 +1,126 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const tag = ctx.params.tag;
+    const category = ctx.params.category;
+    const url = `https://www.gcores.com/categories/${tag + (category ? `?tab=${category}` : '')}`;
+    const res = await got({
+        method: 'get',
+        url,
+    });
+    const data = res.data;
+    const $ = cheerio.load(data);
+    const feedTitle = $('title').text();
+
+    const list = $('.original.am_card.original-normal')
+        .map(function () {
+            const item = {
+                url: $(this).find('a.am_card_content.original_content').attr('href'),
+                title: $(this).find('h3.am_card_title').text(),
+                category: $(this).find('span.original_category>a').text(),
+            };
+            return item;
+        })
+        .get();
+
+    const out = await Promise.all(
+        list.map((item) => {
+            const articleUrl = `https://www.gcores.com${item.url}`;
+
+            return ctx.cache.tryGet(articleUrl, async () => {
+                const itemRes = await got({
+                    method: 'get',
+                    url: articleUrl,
+                });
+
+                const itemPage = itemRes.data;
+                const $ = cheerio.load(itemPage);
+
+                let articleData = await got(`https://www.gcores.com/gapi/v1${item.url}?include=media`);
+
+                articleData = articleData.data.data;
+                let cover;
+                if (articleData.attributes.cover) {
+                    cover = `<img src="https://image.gcores.com/${articleData.attributes.cover}" />`;
+                } else if (articleData.attributes.thumb) {
+                    cover = `<img src="https://image.gcores.com/${articleData.attributes.thumb}" />`;
+                } else {
+                    cover = '';
+                }
+
+                // replace figure with img
+                const articleContent = JSON.parse(articleData.attributes.content);
+                const entityRangeMap = {};
+                for (const block of articleContent.blocks || []) {
+                    if (block.entityRanges.length) {
+                        entityRangeMap[block.key] = block.entityRanges;
+                    }
+                }
+
+                $('figure').each((i, elem) => {
+                    const keyAttr = elem.attribs['data-offset-key'];
+                    const keyMatch = /^(\w+)-(\d+)-(\d)$/.exec(keyAttr);
+                    let actualContent = '';
+                    if (keyMatch) {
+                        const [, key, index] = keyMatch;
+                        if (entityRangeMap[key] && entityRangeMap[key][index]) {
+                            const entityKey = entityRangeMap[key] && entityRangeMap[key][index].key;
+                            const entity = articleContent.entityMap[entityKey];
+                            actualContent = convertEntityToContent(entity);
+                        }
+                    }
+
+                    if (actualContent) {
+                        $(elem).replaceWith(actualContent);
+                    }
+                });
+
+                // remove editor toolbar img
+                $('.md-editor-toolbar').replaceWith('');
+                // remove hidden tip block
+                $('.story_hidden').replaceWith('');
+
+                const content = $('.story.story-show').html();
+                return {
+                    title: item.title,
+                    description: cover + content,
+                    link: articleUrl,
+                    guid: articleUrl,
+                };
+            });
+        })
+    );
+    ctx.state.data = {
+        title: feedTitle,
+        link: url,
+        item: out,
+    };
+};
+
+function convertEntityToContent(entity) {
+    const { type, data } = entity;
+    switch (type) {
+        case 'IMAGE':
+            return `
+<figure>
+<img src="https://image.gcores.com/${data.path}" alt="${data.caption || ''}">
+${data.caption ? `<figcaption>${data.caption}</figcaption>` : ''}
+</figure>`;
+
+        case 'GALLERY':
+            return data.images
+                .map(
+                    (image, i, arr) => `
+<figure>
+<img src="https://image.gcores.com/${image.path}" alt="${image.caption || ''}">
+<figcaption>${data.caption || ''} (${i + 1}/${arr.length}) ${image.caption || ''}</figcaption>
+</figure>
+            `
+                )
+                .join('');
+
+        default:
+            return '';
+    }
+}