rsshub 1.0.0-master.f7cdf8b → 1.0.0-master.f812e91

package/lib/v2/gov/safe/util.js

@@ -0,0 +1,87 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const rootUrl = 'https://www.safe.gov.cn';
+
+const zxfkCategoryApis = {
+    // 业务咨询 https://www.safe.gov.cn/<site>/ywzx/index.html
+    ywzx: 'www/busines/businessQuery?siteid=',
+
+    // 投诉建议 https://www.safe.gov.cn/<site>/tsjy/index.html
+    tsjy: 'www/complaint/complaintQuery?siteid=',
+};
+
+const processZxfkItems = async (site = 'beijing', category = 'ywzx', limit = '3') => {
+    const apiUrl = new URL(`${zxfkCategoryApis[category]}${site}`, rootUrl).href;
+    const currentUrl = new URL(`${site}/${category}/index.html`, rootUrl).href;
+
+    const { data: response } = await got(apiUrl);
+
+    const $ = cheerio.load(response);
+
+    const items = $('#complaint')
+        .slice(0, limit)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const spans = item.find('span[objid]');
+
+            const message = {
+                author: spans.first().text().replace(/:$/, ''),
+                content: spans.eq(1).text(),
+                date: spans.eq(2).text(),
+            };
+
+            const reply = {
+                author: spans.eq(3).text().replace(/:$/, ''),
+                content: spans.eq(4).text(),
+                date: spans.eq(5).text(),
+            };
+
+            return {
+                title: `${message.author}: ${message.content}`,
+                link: new URL(item.find('.title').prop('href'), rootUrl).href,
+                description: art(path.join(__dirname, 'templates/message.art'), {
+                    message,
+                    reply,
+                }),
+                author: `${message.author}/${reply.author}`,
+                guid: item.find('.id').text(),
+                pubDate: parseDate(message.date),
+                updated: parseDate(reply.date),
+            };
+        });
+
+    const { data: currentResponse } = await got(currentUrl);
+
+    const content = cheerio.load(currentResponse);
+
+    const author = content('meta[name="ColumnName"]').prop('content');
+    const subtitle = content('meta[name="ColumnType"]').prop('content');
+
+    const imagePath = 'safe/templateresource/372b1dfdab204181b9b4f943a8e926a6';
+    const image = new URL(`${imagePath}/logo_06.png`, rootUrl).href;
+    const icon = new URL(`${imagePath}/safe.ico`, rootUrl).href;
+
+    return {
+        item: items,
+        title: `${author} - ${subtitle}`,
+        link: currentUrl,
+        description: content('meta[name="ColumnDescription"]').prop('content'),
+        language: 'zh',
+        image,
+        icon,
+        logo: icon,
+        subtitle,
+        author,
+        allowEmpty: true,
+    };
+};
+
+module.exports = {
+    processZxfkItems,
+};

package/lib/v2/gov/zhengce/index.js

@@ -0,0 +1,99 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const { category = 'zuixin' } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 20;
+
+    const rootUrl = 'https://www.gov.cn';
+    const currentUrl = new URL(`zhengce/${category.replace(/\/$/, '')}/`, rootUrl).href;
+
+    const { data: response } = await got(currentUrl);
+
+    const $ = cheerio.load(response);
+
+    let items = $('h4 a, div.subtitle a[title]')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const link = item.prop('href');
+
+            return {
+                title: item.text(),
+                link: link.startsWith('http') ? link : new URL(link, currentUrl).href,
+            };
+        });
+
+    items = await Promise.all(
+        items
+            .filter((item) => /https?:\/\/www\.gov\.cn\/zhengce.*content_\d+\.htm/.test(item.link))
+            .slice(0, limit)
+            .map((item) =>
+                ctx.cache.tryGet(item.link, async () => {
+                    const { data: detailResponse } = await got(item.link);
+
+                    const content = cheerio.load(detailResponse);
+
+                    const processElementText = (el) => content(el).text().split(/：/).pop().trim() || content(el).next().text().trim();
+
+                    const author = content('meta[name="author"]').prop('content');
+
+                    const agencyEl = content('table.bd1')
+                        .find('td')
+                        .toArray()
+                        .filter((a) => content(a).text().startsWith('发文机关'))
+                        .pop();
+
+                    const sourceEl = content('span.font-zyygwj')
+                        .toArray()
+                        .filter((a) => content(a).text().startsWith('来源'))
+                        .pop();
+
+                    const subjectEl = content('table.bd1')
+                        .find('td')
+                        .toArray()
+                        .filter((a) => content(a).text().startsWith('主题分类'))
+                        .pop();
+
+                    const agency = agencyEl ? processElementText(agencyEl) : undefined;
+                    const source = sourceEl ? processElementText(sourceEl) : undefined;
+                    const subject = subjectEl ? processElementText(subjectEl) : content('td.zcwj_ztfl').text();
+
+                    const column = content('meta[name="lanmu"]').prop('content');
+                    const keywords = content('meta[name="keywords"]').prop('content')?.split(/;|,/) ?? [];
+                    const manuscriptId = content('meta[name="manuscriptId"]').prop('content');
+
+                    item.title = content('div.share-title').text() || item.title;
+                    item.description = content('div.TRS_UEDITOR').first().html() || content('div#UCAP-CONTENT, td#UCAP-CONTENT').first().html();
+                    item.author = [agency, source, author].filter((a) => a).join('/');
+                    item.category = [...new Set([subject, column, ...keywords].filter((c) => c))];
+                    item.guid = `gov-zhengce-${manuscriptId}`;
+                    item.pubDate = timezone(parseDate(content('meta[name="firstpublishedtime"]').prop('content'), 'YYYY-MM-DD-HH:mm:ss'), +8);
+                    item.updated = timezone(parseDate(content('meta[name="lastmodifiedtime"]').prop('content'), 'YYYY-MM-DD-HH:mm:ss'), +8);
+
+                    return item;
+                })
+            )
+    );
+
+    const image = new URL($('img.wordlogo').prop('src'), rootUrl).href;
+    const icon = new URL($('link[rel="icon"]').prop('href'), rootUrl).href;
+    const subtitle = $('meta[name="lanmu"]').prop('content');
+    const author = $('div.header_logo a[aria-label]').prop('aria-label');
+
+    ctx.state.data = {
+        item: items,
+        title: author && subtitle ? `${author} - ${subtitle}` : $('title').text(),
+        link: currentUrl,
+        description: $('meta[name="description"]').prop('content'),
+        language: 'zh-CN',
+        image,
+        icon,
+        logo: icon,
+        subtitle,
+        author,
+    };
+};

package/lib/v2/greasyfork/maintainer.js

@@ -2,4 +2,5 @@ module.exports = {
     '/:language/:domain?': ['imlonghao', 'miles170'],
     '/scripts/:script/feedback': ['miles170'],
     '/scripts/:script/versions': ['miles170'],
+    '/scripts/sort/:sort/:language?': ['ingjieye'],
 };

package/lib/v2/greasyfork/router.js

@@ -2,4 +2,5 @@ module.exports = (router) => {
     router.get('/:language/:domain?', require('./scripts'));
     router.get('/scripts/:script/feedback', require('./feedback'));
     router.get('/scripts/:script/versions', require('./versions'));
+    router.get('/scripts/sort/:sort/:language?', require('./scripts'));
 };

package/lib/v2/greasyfork/scripts.js

@@ -1,39 +1,44 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const queryString = require('query-string');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const language = ctx.params.language === 'all' ? 'zh-CN' : ctx.params.language;
+    const language = ctx.params.language === 'all' ? 'zh-CN' : ctx.params.language || 'zh-CN';
     const domain = ctx.params.domain;
     const filter_locale = ctx.params.language === 'all' ? 0 : 1;
-    const url = domain ? `by-site/${domain}` : '';
-    const currentUrl = `https://greasyfork.org/${language}/scripts/${url}`;
+    const sort = ctx.params.sort ? ctx.params.sort : 'updated';
+    const url = domain ? `/by-site/${domain}` : '';
+    const currentUrl = `https://greasyfork.org/${language}/scripts${url}`;
     const res = await got({
         method: 'get',
         url: currentUrl,
-        searchParams: queryString.stringify({
+        searchParams: {
             filter_locale,
-            sort: 'updated',
-        }),
+            sort,
+        },
     });
     const $ = cheerio.load(res.data);
-    const list = $('.script-list').find('h2');
+    const list = $('.script-list').find('article');
 
     ctx.state.data = {
         title: $('title').first().text(),
         link: currentUrl,
         description: $('meta[name=description]').attr('content'),
-        item:
-            list &&
-            list
-                .map((_, item) => {
-                    item = $(item);
-                    return {
-                        title: item.find('a').text(),
-                        description: item.find('.description').text(),
-                        link: item.find('a').attr('href'),
-                    };
-                })
-                .get(),
+        item: list?.toArray().map((item) => {
+            item = $(item);
+            const h2 = item.find('h2');
+            return {
+                title: h2.find('a').text(),
+                description: h2.find('.description').text(),
+                link: new URL(h2.find('a').attr('href'), 'https://greasyfork.org').href,
+                pubDate: parseDate(item.find('.script-list-created-date relative-time').attr('datetime')),
+                updated: parseDate(item.find('.script-list-updated-date relative-time').attr('datetime')),
+                author: item
+                    .find('.script-list-author a')
+                    .toArray()
+                    .map((a) => $(a).text())
+                    .join(', '),
+            };
+        }),
     };
 };

package/lib/{routes/universities → v2}/harvard/health/blog.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const rootUrl = 'https://www.health.harvard.edu';
@@ -12,17 +13,16 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('.text-2xl')
-        .slice(0, 10)
-        .map((_, item) => {
+    const list = $('.lg\\:text-2xl')
+        .toArray()
+        .map((item) => {
             item = $(item).parent();
 
             return {
                 title: item.text(),
                 link: item.attr('href'),
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         list.map((item) =>
@@ -33,10 +33,11 @@ module.exports = async (ctx) => {
                 });
                 const content = cheerio.load(detailResponse.data);
 
-                const date = item.link.substr(item.link.length - 12, 8);
+                const ldJson = JSON.parse(content('script[type="application/ld+json"]').text())['@graph'].find((i) => i['@type'] === 'Article');
 
                 item.description = content('.content-repository-content').html();
-                item.pubDate = Date.parse(`${date.substr(0, 4)}-${date.substr(4, 2)}-${date.substr(6, 2)}`);
+                item.pubDate = parseDate(ldJson.datePublished);
+                item.author = ldJson.author.name;
 
                 return item;
             })

package/lib/v2/harvard/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/health/blog': ['nczitzk'],
+};

package/lib/v2/harvard/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'harvard.edu': {
+        _name: 'Harvard University',
+        'www.health': [
+            {
+                title: 'Health Blog',
+                docs: 'https://docs.rsshub.app/routes/new-media#harvard-health-publishing',
+                source: ['/blog'],
+                target: '/harvard/health/blog',
+            },
+        ],
+    },
+};

package/lib/v2/harvard/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/health/blog', require('./health/blog'));
+};

package/lib/v2/hellobtc/maintainer.js

@@ -2,5 +2,4 @@ module.exports = {
     '/information/:channel?': ['Fatpandac'],
     '/news': ['Fatpandac'],
     '/kepu/:channel?': ['Fatpandac'],
-    '/topic/:id': ['Fatpandac'],
 };

package/lib/v2/hellobtc/radar.js

@@ -74,12 +74,6 @@ module.exports = {
                 source: ['/kepu.html'],
                 target: '/hellobtc/kepu/public_blockchain',
             },
-            {
-                title: '专栏',
-                docs: 'https://docs.rsshub.app/routes/new-media#bai-hua-qu-kuai-lian',
-                source: ['/topic/:id'],
-                target: (params) => `/hellobtc/topic/${params.id.split('.')[0]}`,
-            },
         ],
     },
 };

package/lib/v2/hellobtc/router.js

@@ -2,5 +2,4 @@ module.exports = function (router) {
     router.get('/information/:channel?', require('./information'));
     router.get('/news', require('./news'));
     router.get('/kepu/:channel?', require('./kepu'));
-    router.get('/topic/:id', require('./topic'));
 };

package/lib/v2/huxiu/briefColumn.js

@@ -1,41 +1,29 @@
 const got = require('@/utils/got');
-const { parseDate } = require('@/utils/parse-date');
-const { baseUrl, briefApi, ProcessFeed } = require('./utils');
+
+const { apiBriefRootUrl, processItems, fetchBriefColumnData } = require('./util');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const link = `${briefApi}/briefColumn/getContentListByCategoryId`;
-    const { data: response } = await got
-        .post(link, {
-            form: {
-                platform: 'www',
-                brief_column_id: id,
-            },
-        })
-        .json();
-    const {
-        data: { data: briefDetail },
-    } = await got.post(`${briefApi}/briefColumn/detail`, {
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 20;
+
+    const apiUrl = new URL('briefColumn/getContentListByCategoryId', apiBriefRootUrl).href;
+
+    const { data: response } = await got.post(apiUrl, {
         form: {
             platform: 'www',
             brief_column_id: id,
+            pagesize: limit,
         },
     });
 
-    const list = response.datalist.map((item) => ({
-        title: item.title,
-        link: `${baseUrl}/brief/${item.brief_id}`,
-        description: item.preface,
-        pubDate: parseDate(item.publish_time, 'X'),
-    }));
+    ctx.state.json = response.data.datalist;
+
+    const items = await processItems(response.data.datalist, limit, ctx.cache.tryGet);
 
-    const items = await ProcessFeed(list, ctx.cache);
+    const data = await fetchBriefColumnData(id);
 
     ctx.state.data = {
-        title: `虎嗅 - ${briefDetail.name}`,
-        description: briefDetail.summary,
-        image: briefDetail.head_img,
-        link: `${baseUrl}/briefColumn/${id}.html`,
         item: items,
+        ...data,
     };
 };

package/lib/v2/huxiu/channel.js

@@ -0,0 +1,28 @@
+const got = require('@/utils/got');
+
+const { rootUrl, apiArticleRootUrl, processItems, fetchData } = require('./util');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 20;
+
+    const apiUrl = new URL(`web/${id ? 'channel' : 'article'}/articleList`, apiArticleRootUrl).href;
+    const currentUrl = new URL(id ? `channel/${id}.html` : 'article', rootUrl).href;
+
+    const { data: response } = await got.post(apiUrl, {
+        form: {
+            platform: 'www',
+            channel_id: id,
+            pagesize: limit,
+        },
+    });
+
+    const items = await processItems(response.data?.dataList ?? response.data.datalist, limit, ctx.cache.tryGet);
+
+    const data = await fetchData(currentUrl);
+
+    ctx.state.data = {
+        item: items,
+        ...data,
+    };
+};

package/lib/v2/huxiu/club.js

@@ -0,0 +1,30 @@
+const got = require('@/utils/got');
+
+const { apiBriefRootUrl, processItems, fetchClubData } = require('./util');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 20;
+
+    const apiUrl = new URL('club/briefList', apiBriefRootUrl).href;
+
+    const { data, briefColumnId } = await fetchClubData(id);
+
+    const { data: response } = await got.post(apiUrl, {
+        form: {
+            platform: 'www',
+            club_id: id,
+            brief_column_id: briefColumnId,
+            pagesize: limit,
+        },
+    });
+
+    ctx.state.json = response.data.datalist;
+
+    const items = await processItems(response.data.datalist, limit, ctx.cache.tryGet);
+
+    ctx.state.data = {
+        item: items,
+        ...data,
+    };
+};

package/lib/v2/huxiu/collection.js

@@ -1,39 +1,27 @@
 const got = require('@/utils/got');
-const cheerio = require('cheerio');
-const utils = require('./utils');
-const { parseDate } = require('@/utils/parse-date');
+
+const { rootUrl, apiArticleRootUrl, processItems, fetchData } = require('./util');
 
 module.exports = async (ctx) => {
     const { id } = ctx.params;
-    const link = `${utils.baseUrl}/collection/${id}.html`;
-    const { data: response } = await got(link, {
-        https: {
-            rejectUnauthorized: false,
-        },
-    });
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 10;
 
-    const $ = cheerio.load(response);
+    const apiUrl = new URL('web/collection/articleList', apiArticleRootUrl).href;
+    const currentUrl = new URL(`collection/${id}.html`, rootUrl).href;
 
-    const initialState = utils.parseInitialState($);
-
-    const { collectionDetail } = initialState.collectionDetail;
-    const list = collectionDetail.article_list.datalist.map((e) => ({
-        title: e.title,
-        link: `${utils.baseUrl}/article/${e.aid}.html`,
-        description: e.summary,
-        pubDate: parseDate(e.dateline, 'X'),
-        author: e.user_info.username,
-    }));
+    const { data: response } = await got.post(apiUrl, {
+        form: {
+            platform: 'www',
+            collection_id: id,
+        },
+    });
 
-    const items = await utils.ProcessFeed(list, ctx.cache);
+    const items = await processItems(response.data.datalist, limit, ctx.cache.tryGet);
 
-    const info = `虎嗅 - ${collectionDetail.name}`;
+    const data = await fetchData(currentUrl);
 
     ctx.state.data = {
-        title: info,
-        description: collectionDetail.summary,
-        image: collectionDetail.icon,
-        link,
         item: items,
+        ...data,
     };
 };

package/lib/v2/huxiu/maintainer.js

@@ -1,8 +1,11 @@
 module.exports = {
-    '/article': ['HenryQW'],
-    '/author/:id': ['HenryQW'],
-    '/collection/:id': ['AlexdanerZe'],
+    '/article': ['HenryQW', 'nczitzk'],
+    '/briefcolumn/:id': ['Fatpandac', 'nczitzk'],
+    '/channel/:id?': ['nczitzk'],
+    '/club/:id': ['nczitzk'],
+    '/collection/:id': ['AlexdanerZe', 'nczitzk'],
+    '/member/:id/:type?': ['HenryQW', 'nczitzk'],
     '/moment': ['nczitzk'],
-    '/tag/:id': ['xyqfer', 'HenryQW'],
-    '/search/:keyword': ['xyqfer', 'HenryQW'],
+    '/search/:keyword': ['xyqfer', 'HenryQW', 'nczitzk'],
+    '/tag/:id': ['xyqfer', 'HenryQW', 'nczitzk'],
 };

package/lib/v2/huxiu/member.js

@@ -0,0 +1,27 @@
+const got = require('@/utils/got');
+
+const { rootUrl, apiMemberRootUrl, processItems, fetchData } = require('./util');
+
+module.exports = async (ctx) => {
+    const { id, type = 'article' } = ctx.params;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 10;
+
+    const apiUrl = new URL(`web/${type}/${type}List`, apiMemberRootUrl).href;
+    const currentUrl = new URL(`member/${id}${type === 'article' ? '' : `/${type}`}.html`, rootUrl).href;
+
+    const { data: response } = await got.post(apiUrl, {
+        form: {
+            platform: 'www',
+            uid: id,
+        },
+    });
+
+    const items = await processItems(response.data.datalist, limit, ctx.cache.tryGet);
+
+    const data = await fetchData(currentUrl);
+
+    ctx.state.data = {
+        item: items,
+        ...data,
+    };
+};

package/lib/v2/huxiu/moment.js

@@ -1,39 +1,25 @@
 const got = require('@/utils/got');
-const { parseDate } = require('@/utils/parse-date');
-const { art } = require('@/utils/render');
-const path = require('path');
-const { baseUrl: rootUrl, momentApi: apiRootUrl } = require('./utils');
+
+const { rootUrl, apiMomentRootUrl, processItems, fetchData } = require('./util');
 
 module.exports = async (ctx) => {
-    const currentUrl = `${rootUrl}/moment`;
-    const apiUrl = `${apiRootUrl}/web-v2/moment/feed`;
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit, 10) : 20;
+
+    const apiUrl = new URL('web-v2/moment/feed', apiMomentRootUrl).href;
+    const currentUrl = new URL('moment', rootUrl).href;
 
-    const response = await got({
-        method: 'post',
-        url: apiUrl,
+    const { data: response } = await got.post(apiUrl, {
         form: {
-            last_dateline: parseInt(new Date().getTime() / 1000),
             platform: 'www',
-            is_ai: 0,
         },
     });
 
-    const items = response.data.data.moment_list.datalist[0].datalist.map((item) => ({
-        title: item.content,
-        link: item.share_url,
-        author: item.user_info.username,
-        pubDate: parseDate(item.publish_time * 1000),
-        description: art(path.join(__dirname, 'templates/moment.art'), {
-            description: item.content,
-            images: item.img_urls,
-            video: item.video,
-            comments: item.comment,
-        }),
-    }));
+    const items = await processItems(response.data.moment_list.datalist[0].datalist, limit, ctx.cache.tryGet);
+
+    const data = await fetchData(currentUrl);
 
     ctx.state.data = {
-        title: '虎嗅 - 24小时',
-        link: currentUrl,
         item: items,
+        ...data,
     };
 };