rsshub 1.0.0-master.fe50f9c → 1.0.0-master.fe560d2

package/lib/v2/foresightnews/index.js

@@ -1,6 +1,7 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const zlib = require('zlib');
 
 const categories = {
     '': {
@@ -28,12 +29,11 @@ module.exports = async (ctx) => {
     const rootUrl = 'https://foresightnews.pro';
     const currentUrl = `${rootUrl}/v1/${categories[category].id}?page=1&size=${ctx.query.limit ?? 30}${category === 'column' ? `&column_id=${id}` : ''}`;
 
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
+    const response = await got(currentUrl);
 
-    let items = response.data.data.list.map((item) =>
+    const list = JSON.parse(zlib.inflateSync(Buffer.from(response.data.data.list, 'base64')).toString());
+
+    let items = list.map((item) =>
         item.source_type
             ? {
                   type: item.source_type,
@@ -47,8 +47,8 @@ module.exports = async (ctx) => {
             : {
                   type: category,
                   title: item.title,
-                  author: item?.author?.username ?? undefined,
-                  link: `${rootUrl}/${category === 'article' ? 'article' : 'news'}/detail/${item.id}`,
+                  author: item.author?.username ?? undefined,
+                  link: `${rootUrl}/${category === 'article' || category === 'column' ? 'article' : 'news'}/detail/${item.id}`,
                   category: item.tags ? item.tags.map((tag) => tag.name) : undefined,
                   description: item.content ?? '',
                   pubDate: parseDate(item.publishDate),
@@ -63,10 +63,7 @@ module.exports = async (ctx) => {
         items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 if (item.type !== 'news') {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                    });
+                    const detailResponse = await got(item.link);
 
                     const content = cheerio.load(detailResponse.data);
 

package/lib/v2/foreverblog/feeds.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const currentUrl = 'https://www.foreverblog.cn/feeds.html';
+
+    const response = await got(currentUrl);
+
+    const $ = cheerio.load(response.data);
+    const $articles = $('article[class="post post-type-normal"]');
+    const items = $articles
+        .map((_, el) => {
+            const $titleDiv = $(el).find('h1[class="post-title"]');
+            const title = $titleDiv.text().trim();
+            const link = $titleDiv.find('a').eq(0).attr('href');
+            const author = $(el).find('div[class="post-author"]').text().trim();
+            const postDate = $(el).find('time').text().trim();
+            const pubDate = timezone(parseDate(postDate, 'MM-DD'), +8);
+            const description = `${author}: ${title}`;
+            return {
+                title: description,
+                description,
+                link,
+                pubDate,
+            };
+        })
+        .toArray();
+
+    ctx.state.data = {
+        title: '十年之约——专题展示',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/foreverblog/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/feeds': ['7Wate', 'a180285'],
+};

package/lib/v2/foreverblog/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'foreverblog.cn': {
+        _name: '十年之约',
+        www: [
+            {
+                title: '专题展示',
+                docs: 'https://docs.rsshub.app/blog.html#shi-nian-zhi-yue',
+                source: ['/feeds.html'],
+                target: '/foreverblog/feeds',
+            },
+        ],
+    },
+};

package/lib/v2/foreverblog/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/feeds', require('./feeds'));
+};

package/lib/v2/freewechat/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/profile/:id': ['TonyRL'],
+};

package/lib/v2/freewechat/profile.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { fixArticleContent } = require('@/utils/wechat-mp');
+const baseUrl = 'https://freewechat.com';
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const url = `${baseUrl}/profile/${id}`;
+    const { data: response } = await got(url);
+    const $ = cheerio.load(response);
+    const author = $('h2').text().trim();
+
+    const list = $('.main')
+        .toArray()
+        .slice(0, -1) // last item is a template
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').text(),
+                author,
+                link: `${baseUrl}${item.find('a').attr('href')}`,
+                description: item.find('.preview').text(),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link, {
+                    headers: {
+                        Referer: url,
+                    },
+                });
+                const $ = cheerio.load(response.data);
+
+                $('.js_img_placeholder').remove();
+                $('amp-img').each((_, e) => {
+                    e = $(e);
+                    e.replaceWith(`<img src="${new URL(e.attr('src'), response.url).href}" width="${e.attr('width')}" height="${e.attr('height')}" decoding="async">`);
+                });
+                $('amp-video').each((_, e) => {
+                    e = $(e);
+                    e.replaceWith(`<video width="${e.attr('width')}" height="${e.attr('height')}" controls poster="${e.attr('poster')}">${e.html()}</video>`);
+                });
+
+                item.description = fixArticleContent($('#js_content'));
+                item.pubDate = timezone(parseDate($('#publish_time').text()), +8);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        image: 'https://freewechat.com/favicon.ico',
+        item: items,
+    };
+};

package/lib/v2/freewechat/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'freewechat.com': {
+        _name: '自由微信',
+        '.': [
+            {
+                title: '公众号',
+                docs: 'https://docs.rsshub.app/new-media.html#zi-you-wei-xin',
+                source: ['/profile/:id'],
+                target: '/freewechat/profile/:id',
+            },
+        ],
+    },
+};

package/lib/v2/freewechat/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/profile/:id', require('./profile'));
+};

package/lib/v2/ft/maintainer.js

@@ -1,4 +1,4 @@
 module.exports = {
-    '/ft/myft/:key': ['HenryQW'],
-    '/ft/:language/:channel?': ['HenryQW', 'xyqfer'],
+    '/myft/:key': ['HenryQW'],
+    '/:language/:channel?': ['HenryQW', 'xyqfer'],
 };

package/lib/v2/ft/utils.js

@@ -2,9 +2,9 @@ const got = require('@/utils/got');
 const parser = require('@/utils/rss-parser');
 const cheerio = require('cheerio');
 
-const ProcessFeed = ($, link) => {
+const ProcessFeed = (i, $, link) => {
     const title = $('h1').text();
-    let content = $('div.story-container');
+    let content = $('div.story-container').eq(i);
 
     // 处理封面图片
     content.find('div.story-image > figure').each((_, e) => {
@@ -59,17 +59,20 @@ const getData = async ({ site = 'www', channel, ctx }) => {
     }
 
     const items = await Promise.all(
-        feed.items.splice(0, 10).map((item) => {
+        feed.items.map((item) => {
             item.link = item.link.replace('http://', 'https://');
             return ctx.cache.tryGet(item.link, async () => {
                 const response = await got.get(`${item.link}?full=y&archive`);
 
                 const $ = cheerio.load(response.data);
-                const result = ProcessFeed($, item.link);
+                const results = [];
+                for (let i = 0; i < $('div.story-container').length; i++) {
+                    results.push(ProcessFeed(i, $, item.link));
+                }
 
-                item.title = result.title;
-                item.description = result.content;
-                item.author = result.author;
+                item.title = results[0].title;
+                item.description = results.map((result) => result.content).join('');
+                item.author = results[0].author;
                 return item;
             });
         })

package/lib/v2/gcores/category.js

@@ -12,16 +12,26 @@ module.exports = async (ctx) => {
     const $ = cheerio.load(data);
     const feedTitle = $('title').text();
 
-    const list = $('.original.am_card.original-normal')
-        .map(function () {
+    let list;
+    if (category === 'news') {
+        list = $('a.news').map(function () {
+            const item = {
+                url: $(this).attr('href'),
+                title: $(this).find('.news_content>h3').text(),
+            };
+            return item;
+        });
+    } else {
+        list = $('.original.am_card.original-normal').map(function () {
             const item = {
                 url: $(this).find('.am_card_inner>a').attr('href'),
                 title: $(this).find('h3.am_card_title').text(),
                 category: $(this).find('span.original_category>a').text(),
             };
             return item;
-        })
-        .get();
+        });
+    }
+    list = list.get();
 
     if (list.length > 0 && list.every((item) => item.url === undefined)) {
         throw new Error('Article URL not found! Please submit an issue on GitHub.');
@@ -85,13 +95,13 @@ module.exports = async (ctx) => {
                 $('.story_hidden').replaceWith('');
 
                 const content = $('.story.story-show').html();
-                return {
+                const basicItem = {
                     title: item.title,
                     description: cover + content,
-                    category: item.category,
                     link: articleUrl,
                     guid: articleUrl,
                 };
+                return category === 'news' ? basicItem : { ...basicItem, category: item.category };
             });
         })
     );

package/lib/v2/gelonghui/home.js

@@ -0,0 +1,31 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { tag = 'web_home_page' } = ctx.params;
+    const apiUrl = `https://www.gelonghui.com/api/channels/${tag}/articles/v8`;
+    const { data } = await got(apiUrl);
+
+    const list = data.result.map((article) => {
+        article = article.data;
+        return {
+            title: article.title,
+            description: article.summary,
+            link: article.link,
+            author: article.nick,
+            category: article.source,
+            pubDate: parseDate(article.timestamp, 'X'),
+        };
+    });
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: '格隆汇-财经资讯动态-股市行情',
+        description: '格隆汇为中国投资者出海投资及中国公司出海融资,提供海外投资,港股开户行情,科创板股票发行数据、资讯、研究、交易等一站式服务,目前业务范围主要涉及港股与美股两大市场,未来将陆续开通台湾、日本、印度、欧洲等市场.',
+        image: 'https://cdn.gelonghui.com/static/web/www.ico.la.ico',
+        link: 'https://www.gelonghui.com',
+        item: items,
+    };
+};

package/lib/v2/gelonghui/keyword.js

@@ -0,0 +1,34 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { keyword } = ctx.params;
+    const currentUrl = `https://www.gelonghui.com/api/post/search/v4`;
+    const { data } = await got(currentUrl, {
+        searchParams: {
+            keyword,
+            isVipArticle: false,
+            count: ctx.query.limit ? Number(ctx.query.limit) : 20,
+            page: 1,
+            searchArea: 'all',
+        },
+    });
+
+    const list = data.result.map((item) => ({
+        title: item.title,
+        description: item.summary,
+        link: item.link,
+        author: item.source,
+        pubDate: parseDate(item.timestamp, 'X'),
+    }));
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `格隆汇 - 关键词 “${keyword}” 的文章`,
+        link: currentUrl,
+        item: items,
+        description: `找到关于 “ ${keyword} ”的文章，共${data.totalCount}个结果`,
+    };
+};

package/lib/v2/gelonghui/maintainer.js

@@ -0,0 +1,6 @@
+module.exports = {
+    '/home/:tag?': ['TonyRL'],
+    '/keyword/:keyword': ['nczitzk'],
+    '/subject/:id': ['nczitzk'],
+    '/user/:id': ['nczitzk'],
+};

package/lib/v2/gelonghui/radar.js

@@ -0,0 +1,31 @@
+module.exports = {
+    'gelonghui.com': {
+        _name: '格隆汇',
+        '.': [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/finance.html#ge-long-hui',
+                source: ['/tag/:tag', '/'],
+                target: (params) => `/gelonghui/home${params.tag ? `/${params.tag}` : ''}`,
+            },
+            {
+                title: '搜索关键字',
+                docs: 'https://docs.rsshub.app/finance.html#ge-long-hui',
+                source: ['/search'],
+                target: (_, url) => `/gelonghui/keyword/${new URL(url).searchParams.get('keyword')}`,
+            },
+            {
+                title: '主题文章',
+                docs: 'https://docs.rsshub.app/finance.html#ge-long-hui',
+                source: ['/subject/:id'],
+                target: '/gelonghui/subject/:id',
+            },
+            {
+                title: '用户文章',
+                docs: 'https://docs.rsshub.app/finance.html#ge-long-hui',
+                source: ['/user/:id'],
+                target: '/gelonghui/user/:id',
+            },
+        ],
+    },
+};

package/lib/v2/gelonghui/router.js

@@ -0,0 +1,6 @@
+module.exports = (router) => {
+    router.get('/home/:tag?', require('./home'));
+    router.get('/keyword/:keyword', require('./keyword'));
+    router.get('/subject/:id', require('./subject'));
+    router.get('/user/:id', require('./user'));
+};

package/lib/v2/gelonghui/subject.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const apiUrl = `https://www.gelonghui.com/api/subjects/${id}/contents`;
+    const pageUrl = `https://www.gelonghui.com/subject/${id}`;
+    const { data: response } = await got(pageUrl);
+    const { data } = await got(apiUrl, {
+        searchParams: {
+            isChoice: false,
+        },
+    });
+
+    const $ = cheerio.load(response);
+
+    const list = data.result.map((item) => ({
+        title: item.title,
+        description: item.summary,
+        link: item.link,
+        author: item.source,
+        pubDate: parseDate(item.timestamp, 'X'),
+    }));
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `格隆汇 - 主题 ${$('span.user-nick').text()} 的文章`,
+        description: $('div.user-name').parent().children('p').text(),
+        image: $('.subject-list-title').find('img').attr('data-src'),
+        link: `https://www.gelonghui.com/subject/${id}`,
+        item: items,
+    };
+};

package/lib/v2/gelonghui/user.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { id: userId } = ctx.params;
+    const currentUrl = 'https://www.gelonghui.com/api/community/dynamic/my-dynamics/v2';
+    const { data } = await got(currentUrl, {
+        searchParams: {
+            article: true,
+            userId,
+            count: ctx.query.limit ? Number(ctx.query.limit) : 15,
+        },
+    });
+
+    const list = data.result.map((item) => ({
+        title: item.title,
+        link: item.route,
+        description: item.content,
+        author: item.user.nick,
+        pubDate: parseDate(item.createTimestamp, 'X'),
+    }));
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `格隆汇 - 用户 ${data.result[0].user.nick} 的文章`,
+        description: data.result.find((i) => i.user).user.brief,
+        image: data.result.find((i) => i.user).user.avatar.split('@')[0],
+        link: data.result.find((i) => i.user).user.route.replace('https://m.gelonghui.com', 'https://www.gelonghui.com'),
+        item: items,
+    };
+};

package/lib/v2/gelonghui/utils.js

@@ -0,0 +1,21 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const parseItem = (item, tryGet) =>
+    tryGet(item.link, async () => {
+        const { data: res } = await got(item.link);
+        const $ = cheerio.load(res);
+        if (item.link.startsWith('https://www.gelonghui.com/live/')) {
+            item.title = $('.type-name').next().text().trim();
+            item.description = $('.dtb-content').html();
+        } else {
+            // article
+            item.title = $('.article-title').text().trim();
+            item.description = $('.article-summary').html() + $('article.article-with-html').html();
+        }
+        return item;
+    });
+
+module.exports = {
+    parseItem,
+};

package/lib/v2/github/trending.js

@@ -1,59 +1,91 @@
+const config = require('@/config').value;
 const got = require('@/utils/got');
 const { art } = require('@/utils/render');
 const cheerio = require('cheerio');
 const path = require('path');
 
 module.exports = async (ctx) => {
+    if (!config.github || !config.github.access_token) {
+        throw 'GitHub trending RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>';
+    }
     const since = ctx.params.since;
     const language = ctx.params.language === 'any' ? '' : ctx.params.language;
     const spoken_language = ctx.params.spoken_language ?? '';
-    const url = `https://github.com/trending/${encodeURIComponent(language)}?since=${since}&spoken_language_code=${spoken_language}`;
 
-    const response = await got({
+    const trendingUrl = `https://github.com/trending/${encodeURIComponent(language)}?since=${since}&spoken_language_code=${spoken_language}`;
+    const { data: trendingPage } = await got({
         method: 'get',
-        url,
+        url: trendingUrl,
         headers: {
-            Referer: url,
+            Referer: trendingUrl,
         },
     });
+    const $ = cheerio.load(trendingPage);
 
-    const data = response.data;
-
-    const $ = cheerio.load(data);
-    const list = $('article');
-
-    const items = await Promise.all(
-        list.map((_, item) => {
-            item = $(item);
-            const endpoint = item.find('h1 a').attr('href');
-            const link = `https://github.com${endpoint}`;
-            return ctx.cache.tryGet(`github:trending:${endpoint}`, async () => {
-                const response = await got(link);
+    const articles = $('article');
+    const trendingRepos = articles
+        .map((_, item) => {
+            const [owner, name] = $(item).find('h1').text().split('/');
+            return {
+                name: name.trim(),
+                owner: owner.trim(),
+            };
+        })
+        .get();
 
-                const $ = cheerio.load(response.data);
-                const cover = $('meta[property="og:image"]');
+    const { data: repoData } = await got({
+        method: 'post',
+        url: 'https://api.github.com/graphql',
+        headers: {
+            Authorization: `bearer ${config.github.access_token}`,
+        },
+        json: {
+            query: `
+            query {
+            ${trendingRepos
+                .map(
+                    (repo, index) => `
+                _${index}: repository(owner: "${repo.owner}", name: "${repo.name}") {
+                    ...RepositoryFragment
+                }
+            `
+                )
+                .join('\n')}
+            }
 
-                const single = {
-                    title: item.find('h1').text(),
-                    author: item.find('h1').text().split('/')[0].trim(),
-                    description: art(path.join(__dirname, 'templates/trending-description.art'), {
-                        cover: cover.attr('content'),
-                        desc: item.find('p').text(),
-                        lang: item.find('span[itemprop="programmingLanguage"]').text() || 'Unknown',
-                        stars: item.find('.Link--muted').eq(0).text().trim(),
-                        forks: item.find('.Link--muted').eq(1).text().trim(),
-                    }),
-                    link,
-                };
+            fragment RepositoryFragment on Repository {
+                description
+                forkCount
+                nameWithOwner
+                openGraphImageUrl
+                primaryLanguage {
+                    name
+                }
+                stargazerCount
+            }
+            `,
+        },
+    });
 
-                return single;
-            });
-        })
-    );
+    const repos = Object.values(repoData.data).map((repo) => {
+        const found = trendingRepos.find((r) => `${r.owner}/${r.name}` === repo.nameWithOwner);
+        return { ...found, ...repo };
+    });
 
     ctx.state.data = {
         title: $('title').text(),
-        link: url,
-        item: items,
+        link: trendingUrl,
+        item: repos.map((r) => ({
+            title: r.nameWithOwner,
+            author: r.owner,
+            description: art(path.join(__dirname, 'templates/trending-description.art'), {
+                cover: r.openGraphImageUrl,
+                desc: r.description,
+                forks: r.forkCount,
+                lang: r.primaryLanguage?.name || 'Unknown',
+                stars: r.stargazerCount,
+            }),
+            link: `https://github.com/${r.nameWithOwner}`,
+        })),
     };
 };