rsshub 1.0.0-master.f7add90 → 1.0.0-master.f7bfacc

package/lib/v2/flyert/utils.js

@@ -16,19 +16,25 @@ async function load(link) {
     $('script, style').remove();
     $('#loginDialog').remove();
 
+    // 获取第一个帖子对象
+    const firstpost = $('.firstpost');
+
     // 修改图片中的链接
-    $('ignore_js_op img').each(function () {
-        const new_src = $('ignore_js_op img').attr('file');
-        $(this).attr('src', new_src);
+    firstpost.find('ignore_js_op img').each(function () {
+        $(this).attr('src', $(this).attr('file'));
+        // remove useless atrributes
+        for (const attr of ['id', 'aid', 'zoomfile', 'file', 'zoomfile', 'class', 'onclick', 'title', 'inpost', 'alt', 'onmouseover']) {
+            $(this).removeAttr(attr);
+        }
     });
 
     // 去除全文中图片的多余标签
-    const images = $('ignore_js_op img');
-    $('ignore_js_op').remove();
-    $("[class*='post_message']").append(images);
+    const images = firstpost.find('ignore_js_op img');
+    firstpost.find('ignore_js_op').remove();
+    firstpost.append(images);
 
-    // 提取内容
-    const description = $("[class*='post_message']").html();
+    // // 提取内容
+    const description = firstpost.html();
 
     return { description };
 }

package/lib/v2/followin/index.js

@@ -0,0 +1,35 @@
+const got = require('@/utils/got');
+const { apiUrl, favicon, getBParam, getBuildId, getGToken, parseList, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { categoryId = '1', lang = 'en' } = ctx.params;
+    const { limit = 20 } = ctx.query;
+    const gToken = await getGToken(ctx.cache.tryGet);
+    const bParam = getBParam(lang);
+
+    const { data: response } = await got.post(`${apiUrl}/feed/list/recommended`, {
+        headers: {
+            'x-bparam': JSON.stringify(bParam),
+            'x-gtoken': gToken,
+        },
+        json: {
+            category_id: parseInt(categoryId),
+            count: parseInt(limit),
+        },
+    });
+    if (response.code !== 2000) {
+        throw Error(response.msg);
+    }
+
+    const buildId = await getBuildId(ctx.cache.tryGet);
+
+    const list = parseList(response.data.list, lang, buildId);
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: 'Followin',
+        link: 'https://followin.io',
+        image: favicon,
+        item: items,
+    };
+};

package/lib/v2/followin/kol.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const { baseUrl, getBuildId, parseList, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { kolId, lang = 'en' } = ctx.params;
+    const { limit = 10 } = ctx.query;
+
+    const buildId = await getBuildId(ctx.cache.tryGet);
+    const { data: response } = await got(`${baseUrl}/_next/data/${buildId}/${lang}/kol/${kolId}.json`);
+
+    const { queries } = response.pageProps.dehydratedState;
+    const { data: profile } = queries.find((q) => q.queryKey[0] === '/user/get_profile').state;
+
+    const list = parseList(queries.find((q) => q.queryKey[0] === '/feed/list/user').state.data.pages[0].list.slice(0, limit), lang, buildId);
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `${profile.nickname} - Followin`,
+        description: profile.bio,
+        link: `${baseUrl}/${lang}/kol/${kolId}`,
+        image: profile.avatar,
+        language: lang,
+        item: items,
+    };
+};

package/lib/v2/followin/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/kol/:kolId/:lang?': ['TonyRL'],
+    '/news/:lang?': ['TonyRL'],
+    '/tag/:tagId/:lang?': ['TonyRL'],
+    '/topic/:topicId/:lang?': ['TonyRL'],
+    '/:categoryId?/:lang?': ['TonyRL'],
+};

package/lib/v2/followin/news.js

@@ -0,0 +1,21 @@
+const got = require('@/utils/got');
+const { baseUrl, favicon, getBuildId, parseList, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { lang = 'en' } = ctx.params;
+    const { limit = 20 } = ctx.query;
+
+    const buildId = await getBuildId(ctx.cache.tryGet);
+    const { data: response } = await got(`${baseUrl}/_next/data/${buildId}/${lang}/news.json`);
+
+    const list = parseList(response.pageProps.dehydratedState.queries.find((q) => q.queryKey[0] === '/feed/list/recommended/news').state.data.pages[0].list.slice(0, limit), lang, buildId);
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `${lang === 'en' ? 'News' : lang === 'vi' ? 'Bản tin' : '快讯'} - Followin`,
+        link: `${baseUrl}/${lang}/news`,
+        image: favicon,
+        language: lang,
+        item: items,
+    };
+};

package/lib/v2/followin/radar.js

@@ -0,0 +1,37 @@
+module.exports = {
+    'followin.io': {
+        _name: 'Followin',
+        '.': [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/finance.html#followin',
+                source: ['/:lang'],
+                target: (params) => (params.lang !== 'news' ? '/followin/1/:lang' : ''),
+            },
+            {
+                title: '快讯',
+                docs: 'https://docs.rsshub.app/finance.html#followin',
+                source: ['/:lang?/news', '/news'],
+                target: '/followin/news/:lang?',
+            },
+            {
+                title: 'KOL',
+                docs: 'https://docs.rsshub.app/finance.html#followin',
+                source: ['/:lang/kol/:kolId', '/kol/:kolId'],
+                target: '/followin/kol/:kolId/:lang?',
+            },
+            {
+                title: '话题',
+                docs: 'https://docs.rsshub.app/finance.html#followin',
+                source: ['/:lang/topic/:topicId', '/topic/:topicId'],
+                target: '/followin/topic/:topicId/:lang?',
+            },
+            {
+                title: '标签',
+                docs: 'https://docs.rsshub.app/finance.html#followin',
+                source: ['/:lang/tag/:tagId', '/tag/:tagId'],
+                target: '/followin/tag/:tagId/:lang?',
+            },
+        ],
+    },
+};

package/lib/v2/followin/router.js

@@ -0,0 +1,7 @@
+module.exports = (router) => {
+    router.get('/kol/:kolId/:lang?', require('./kol'));
+    router.get('/news/:lang?', require('./news'));
+    router.get('/tag/:tagId/:lang?', require('./tag'));
+    router.get('/topic/:topicId/:lang?', require('./topic'));
+    router.get('/:categoryId?/:lang?', require('./index'));
+};

package/lib/v2/followin/tag.js

@@ -0,0 +1,44 @@
+const got = require('@/utils/got');
+const { apiUrl, baseUrl, getBParam, getBuildId, getGToken, parseList, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { tagId, lang = 'en' } = ctx.params;
+    const { limit = 20 } = ctx.query;
+
+    const buildId = await getBuildId(ctx.cache.tryGet);
+    const tagInfo = await ctx.cache.tryGet(`followin:tag:${tagId}:${lang}`, async () => {
+        const { data: response } = await got(`${baseUrl}/_next/data/${buildId}/${lang}/tag/${tagId}.json`);
+        const { queries } = response.pageProps.dehydratedState;
+        const { base_info: tagInfo } = queries.find((q) => q.queryKey[0] === '/tag/info/v2').state.data;
+        return tagInfo;
+    });
+
+    const gToken = await getGToken(ctx.cache.tryGet);
+    const bParam = getBParam(lang);
+    const { data: tagResponse } = await got.post(`${apiUrl}/feed/list/tag`, {
+        headers: {
+            'x-bparam': JSON.stringify(bParam),
+            'x-gtoken': gToken,
+        },
+        json: {
+            count: limit,
+            id: parseInt(tagId),
+            type: 'tag_discussion_feed',
+        },
+    });
+    if (tagResponse.code !== 2000) {
+        throw Error(tagResponse.msg);
+    }
+
+    const list = parseList(tagResponse.data.list.slice(0, limit), lang, buildId);
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `${tagInfo.name} - Followin`,
+        description: tagInfo.description,
+        link: `${baseUrl}/${lang}/tag/${tagId}`,
+        image: tagInfo.logo,
+        language: lang,
+        item: items,
+    };
+};

package/lib/v2/followin/templates/thread.art

@@ -0,0 +1,15 @@
+{{ each list l }}
+    {{@ l.translated_content.replaceAll('\n', '<br>') || l.content.replaceAll('\n', '<br>') }}
+    <br>
+    {{ if l.images.length }}
+        {{ each l.images img }}
+            <img src="{{ img }}">
+        {{ /each }}
+    {{ /if }}
+    {{ if l.link_previews.length }}
+        {{ each l.link_previews a }}
+            <a href="{{ a.link }}">{{ a.title }}<br>{{ a.content }}</a>
+        {{ /each }}
+    {{ /if }}
+    <br>
+{{ /each }}

package/lib/v2/followin/topic.js

@@ -0,0 +1,25 @@
+const got = require('@/utils/got');
+const { baseUrl, getBuildId, parseList, parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { topicId, lang = 'en' } = ctx.params;
+    const { limit = 20 } = ctx.query;
+
+    const buildId = await getBuildId(ctx.cache.tryGet);
+    const { data: response } = await got(`${baseUrl}/_next/data/${buildId}/${lang}/topic/${topicId}.json`);
+
+    const { queries } = response.pageProps.dehydratedState;
+    const { data: topicInfo } = queries.find((q) => q.queryKey[0] === '/topic/info').state;
+
+    const list = parseList(queries.find((q) => q.queryKey[0] === '/feed/list/topic').state.data.pages[0].list.slice(0, limit), lang, buildId);
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `${topicInfo.title} - Followin`,
+        description: topicInfo.desc,
+        link: `${baseUrl}/${lang}/topic/${topicId}`,
+        image: topicInfo.logo,
+        language: lang,
+        item: items,
+    };
+};

package/lib/v2/followin/utils.js

@@ -0,0 +1,84 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const { art } = require('@/utils/render');
+const { join } = require('path');
+
+const apiUrl = 'https://api.followin.io';
+const baseUrl = 'https://followin.io';
+const favicon = `${baseUrl}/favicon.ico`;
+
+const getBParam = (lang) => ({
+    a: 'web',
+    b: '',
+    c: lang,
+    d: 0,
+    e: '',
+    f: '',
+    g: '',
+    h: '0.1.0',
+    i: 'official',
+});
+
+const getBuildId = (tryGet) =>
+    tryGet(
+        'followin:buildId',
+        async () => {
+            const { data: pageResponse } = await got(baseUrl);
+            const $ = cheerio.load(pageResponse);
+            const { buildId } = JSON.parse($('script#__NEXT_DATA__').text());
+            return buildId;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+const getGToken = (tryGet) =>
+    tryGet('followin:gtoken', async () => {
+        const { data } = await got.post(`${apiUrl}/user/gtoken`);
+        return data.data.gtoken;
+    });
+
+const parseList = (list, lang, buildId) =>
+    list.map((item) => ({
+        title: item.translated_title || item.title,
+        description: item.translated_content || item.content,
+        link: `${baseUrl}/${lang === 'en' ? '' : `${lang}/`}feed/${item.id}`,
+        pubDate: parseDate(item.publish_time, 'x'),
+        category: item.tags.map((tag) => tag.name),
+        author: item.nickname,
+        nextData: `${baseUrl}/_next/data/${buildId}/${lang}/feed/${item.id}.json`,
+    }));
+
+const parseItem = (item, tryGet) =>
+    tryGet(item.link, async () => {
+        const { data } = await got(item.nextData);
+
+        const { queries } = data.pageProps.dehydratedState;
+        const info = queries.find((q) => q.queryKey[0] === '/feed/info').state;
+        const thread = queries.find((q) => q.queryKey[0] === '/feed/thread');
+        if (thread) {
+            item.description = art(join(__dirname, 'templates/thread.art'), {
+                list: thread.state.data.list,
+            });
+        } else {
+            item.description = info.data.translated_full_content || info.data.full_content;
+        }
+
+        item.updated = parseDate(info.dataUpdatedAt, 'x');
+        item.category = [...new Set([...item.category, ...info.data.tags.map((tag) => tag.name)])];
+
+        return item;
+    });
+
+module.exports = {
+    apiUrl,
+    baseUrl,
+    favicon,
+    getBParam,
+    getBuildId,
+    getGToken,
+    parseList,
+    parseItem,
+};

package/lib/v2/freebuf/index.js

@@ -0,0 +1,41 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const { type = 'web' } = ctx.params;
+
+    const fapi = 'https://www.freebuf.com/fapi/frontend/category/list';
+    const baseUrl = 'https://www.freebuf.com';
+    const rssLink = `${baseUrl}/articles/${type}`;
+
+    const options = {
+        headers: {
+            referer: 'https://www.freebuf.com',
+            accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+        },
+        searchParams: {
+            name: type,
+            page: 1,
+            limit: 20,
+            select: 0,
+            order: 0,
+            type: 'category',
+        },
+    };
+
+    const response = await got.get(fapi, options).json();
+
+    const items = response.data.data_list.map((item) => ({
+        title: item.post_title,
+        link: `${baseUrl}${item.url}`,
+        description: item.content,
+        pubDate: parseDate(item.post_date),
+        author: item.nickname,
+    }));
+
+    ctx.state.data = {
+        title: `Freebuf ${type}`,
+        link: rssLink,
+        item: items,
+    };
+};

package/lib/v2/freebuf/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/articles/:type': ['trganda'],
+};

package/lib/v2/freebuf/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'freebuf.com': {
+        _name: 'Freebuf',
+        '.': [
+            {
+                title: '文章',
+                docs: 'https://docs.rsshub.app/blog.html#freebuf',
+                source: ['/articles/:type/*.html', '/articles/:type'],
+                target: '/freebuf/articles/:type',
+            },
+        ],
+    },
+};

package/lib/v2/freebuf/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/articles/:type', require('./index'));
+};

package/lib/v2/freecomputerbooks/index.js

@@ -1,16 +1,20 @@
 const cheerio = require('cheerio');
+const path = require('path');
 
 const got = require('@/utils/got');
+const { art } = require('@/utils/render');
 const { parseDate } = require('@/utils/parse-date');
 
 const baseURL = 'https://freecomputerbooks.com/';
 
+async function cheerioLoad(url) {
+    return cheerio.load((await got(url)).data);
+}
+
 module.exports = async (ctx) => {
     const categoryId = ctx.params.category?.trim();
     const requestURL = categoryId ? new URL(`${categoryId}.html`, baseURL).href : baseURL;
-
-    const response = await got({ method: 'get', url: requestURL });
-    const $ = cheerio.load(response.data);
+    const $ = await cheerioLoad(requestURL);
 
     // As observation has shown that each page only has one element of the
     // class, thus to simplify the processing the text is directly extracted.
@@ -24,23 +28,22 @@ module.exports = async (ctx) => {
 
         // For a "Selected New Books" page, the <ul> element's id. is
         // `newBooksG`; for an ordinary category page, it's `newBooksL`.
-        item: $('ul[id^=newBooks] > li')
-            .toArray()
-            .map((elem) => buildPostItem($(elem), categoryTitle, $)),
+        item: await Promise.all(
+            $('ul[id^=newBooks] > li')
+                .toArray()
+                .map((elem) => buildPostItem($(elem), categoryTitle, ctx.cache))
+        ),
     };
 };
 
-function buildPostItem(listItem, categoryTitle, $) {
-    // Cover in original size, e.g., `covers/xyz_43x55.jpg` -> `covers/xyz.jpg`
-    const image = listItem.find('img');
-    image.attr('src', image.attr('src').replace(/_\d+x\d+/, ''));
+function buildPostItem(listItem, categoryTitle, cache) {
+    const $ = cheerio.load(''); // the only use below doesn't care about the content
 
     const postLink = listItem.find('a:first');
     const postInfo = listItem.find('p:contains("Post under")');
     const postItem = {
         title: postLink.text(),
-        link: postLink.attr('href'),
-        description: listItem.html(),
+        link: new URL(postLink.attr('href'), baseURL).href,
 
         // Only a "Selected New Books" page has exclicit categorization info.
         // for posts; an ordinary category page hasn't, then in which case the
@@ -62,5 +65,30 @@ function buildPostItem(listItem, categoryTitle, $) {
         postItem.pubDate = parseDate(pubDateText);
     }
 
-    return postItem;
+    return cache.tryGet(postItem.link, () => insertDescriptionInto(postItem));
+}
+
+async function insertDescriptionInto(item) {
+    const $ = await cheerioLoad(item.link);
+
+    // Eliminate all comment nodes to avoid their being selected and rendered in
+    // the final output (I know this is actually unnecessary, but please forgive
+    // my mysophobia).
+    $.root()
+        .find('*')
+        .contents()
+        .filter((_, node) => node.type === 'comment')
+        .remove();
+
+    const imageURL = $('#bookdesc img[title]').attr('src');
+    const metadata = $('#booktitle ul').removeAttr('style');
+    const content = $('#bookdesccontent').removeAttr('id');
+
+    metadata.find('li:contains(Share This)').remove();
+    content.find('img[src$="/hot.gif"]').remove();
+    content.find(':contains(Similar Books)').nextAll().addBack().remove();
+
+    item.description = art(path.join(__dirname, 'templates/desc.art'), { imageURL, metadata, content });
+
+    return item;
 }

package/lib/v2/freecomputerbooks/templates/desc.art

@@ -0,0 +1,3 @@
+<figure><img src="{{ imageURL }}"></figure>
+{{@ metadata }}
+{{@ content }}

package/lib/v2/futunn/main.js

@@ -0,0 +1,67 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+module.exports = async (ctx) => {
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 50;
+
+    const rootUrl = 'https://news.futunn.com';
+    const currentUrl = `${rootUrl}/main`;
+    const apiUrl = `${rootUrl}/news-site-api/main/get-market-list?size=${limit}`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let items = response.data.data.list.map((item) => ({
+        title: item.title,
+        link: item.url.split('?')[0],
+        author: item.source,
+        pubDate: parseDate(item.timestamp * 1000),
+        description: art(path.join(__dirname, 'templates/description.art'), {
+            abs: item.abs,
+            pic: item.pic,
+        }),
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                if (/news\.futunn\.com/.test(item.link)) {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+
+                    const content = cheerio.load(detailResponse.data);
+
+                    content('.futu-news-time-stamp').remove();
+                    content('.nnstock').each(function () {
+                        content(this).replaceWith(`<a href="${content(this).attr('href')}">${content(this).text().replace(/\$/g, '')}</a>`);
+                    });
+
+                    item.description = content('.origin_content').html();
+                    item.category = [
+                        ...content('.news__from-topic__title')
+                            .toArray()
+                            .map((a) => content(a).text()),
+                        ...content('#relatedStockWeb .stock-name')
+                            .toArray()
+                            .map((s) => content(s).text().trim()),
+                    ];
+                }
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'Futubull - Headlines',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/futunn/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/main': ['Wsine', 'nczitzk'],
+};

package/lib/v2/futunn/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'futunn.com': {
+        _name: '富途牛牛',
+        news: [
+            {
+                title: '要闻',
+                docs: 'https://docs.rsshub.app/finance.html#fu-tu-niu-niu',
+                source: ['/main', '/'],
+                target: '/futunn/main',
+            },
+        ],
+    },
+};

package/lib/v2/futunn/router.js

@@ -0,0 +1,5 @@
+module.exports = function (router) {
+    router.get('/highlights', require('./main'));
+    router.get('/main', require('./main'));
+    router.get('/', require('./main'));
+};

package/lib/v2/futunn/templates/description.art

@@ -0,0 +1,6 @@
+{{ if pic }}
+<img src="{{ pic }}">
+{{ /if }}
+{{ if abs }}
+<p>{{ abs }}</p>
+{{ /if }}

package/lib/v2/fzmtr/announcements.js

@@ -0,0 +1,38 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const domain = 'www.fzmtr.com';
+    const announcementsUrl = `http://${domain}/html/fzdt/tzgg/index.html`;
+    const response = await got(announcementsUrl);
+    const data = response.data;
+
+    const $ = cheerio.load(data);
+    const list = $('span#resources li')
+        .map((_, item) => {
+            item = $(item);
+            const url = `http://${domain}` + item.find('a').attr('href');
+            const title = item.find('a').text();
+            const publishTime = parseDate(item.find('span').text());
+            return {
+                title,
+                link: url,
+                author: '福州地铁',
+                pubtime: publishTime,
+            };
+        })
+        .get();
+
+    ctx.state.data = {
+        title: '福州地铁通知公告',
+        url: announcementsUrl,
+        description: '福州地铁通知公告',
+        item: list.map((item) => ({
+            title: item.title,
+            pubDate: item.pubtime,
+            link: item.link,
+            author: item.author,
+        })),
+    };
+};

package/lib/v2/fzmtr/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/announcements': ['HankChow'],
+};

package/lib/v2/fzmtr/radar.js

@@ -0,0 +1,11 @@
+module.exports = {
+    'fzmtr.com': {
+        _name: '福州地铁',
+        www: [
+            {
+                title: '通知公告',
+                docs: 'https://docs.rsshub.app/travel.html#fu-zhou-di-tie',
+            },
+        ],
+    },
+};