rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/v2/gov/stats/utils.js

@@ -0,0 +1,73 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const parseContList = async (url, selector, ctx) => {
+    const response = await got(url);
+    const $ = cheerio.load(response.data);
+    const list = $(selector)
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 12)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('.cont_tit03').text(),
+                link: new URL(item.attr('href'), url).href,
+                pubDate: parseDate(item.find('.cont_tit02').text(), 'YYYY-MM-DD'),
+            };
+        })
+        .filter((item) => item.title); // exclude the empty title
+    const title = $('#PL_DAOHANG')
+        .text()
+        .replace(/(.+)首页/, '国家统计局');
+    return { list, title };
+};
+
+const parseXilan = async (item, ctx) =>
+    await ctx.cache.tryGet(item.link, async () => {
+        const response = await got(item.link);
+        const $ = cheerio.load(response.data);
+        const title = $('.xilan_titf').text();
+        item.author = title.match(/来源：(.*)发布时间/)?.[1].trim() ?? '国家统计局';
+        item.pubDate = timezone(parseDate(title.match(/发布时间：(.*)/)?.[1].trim() ?? item.pubDate), +8);
+
+        // Change the img src from relative to absolute for a better compatibility
+        $('.xilan_con')
+            .find('img')
+            .each((_, el) => {
+                $(el).attr('src', new URL($(el).attr('src'), item.link).href);
+                // oldsrc is causing freshrss imageproxy not to work correctly
+                $(el).removeAttr('oldsrc').removeAttr('alt');
+            });
+        item.description = $('.xilan_con').html();
+
+        const attachmentTitle = $('.wenzhang_tit').filter(function () {
+            return $(this).text().trim() === '相关附件';
+        });
+        if (attachmentTitle.length > 0) {
+            const attachments = attachmentTitle
+                .first()
+                .next('.wenzhang_list')
+                .find('a')
+                .toArray()
+                .map((attachment) => {
+                    attachment = $(attachment);
+                    return {
+                        href: new URL(attachment.attr('href'), item.link).href,
+                        text: attachment.text().trim(),
+                    };
+                });
+            item.description += art(path.join(__dirname, 'templates/attachments.art'), {
+                attachments,
+            });
+        }
+        return item;
+    });
+
+module.exports = {
+    parseXilan,
+    parseContList,
+};

package/lib/v2/gov/xinyi/xinyi.js

@@ -0,0 +1,19 @@
+const { gdgov } = require('../general/general');
+
+module.exports = async (ctx) => {
+    const info = {
+        defaultPath: 'zwgk/zcjd/',
+        list_element: '.newsList li a',
+        list_include: 'site',
+        title_element: '.title',
+        title_match: '(.*)',
+        description_element: '.conTxt',
+        author_element: undefined,
+        author_match: undefined,
+        authorisme: '信宜市人民政府网',
+        pubDate_element: '.property span:nth-child(-n+2)',
+        pubDate_match: '发布时间：(.*)',
+        pubDate_format: undefined,
+    };
+    await gdgov(info, ctx);
+};

package/lib/v2/gov/zhengce/govall.js

@@ -0,0 +1,55 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { advance } = ctx.params;
+    const link = `http://sousuo.gov.cn/list.htm`;
+    const params = new URLSearchParams({
+        n: 20,
+        t: 'govall',
+        sort: 'pubtime',
+        advance: 'true',
+    });
+    const query = `${params.toString()}&${advance}`;
+    const res = await got.get(link, {
+        searchParams: query.replace(/([\u4e00-\u9fa5])/g, (str) => encodeURIComponent(str)),
+    });
+    const $ = cheerio.load(res.data);
+
+    const list = $('body > div.dataBox > table > tbody > tr')
+        .slice(1)
+        .toArray()
+        .map((elem) => {
+            elem = $(elem);
+            return {
+                title: elem.find('td:nth-child(2) > a').text(),
+                link: elem.find('td:nth-child(2) > a').attr('href'),
+                pubDate: timezone(parseDate(elem.find('td:nth-child(5)').text()), 8),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                let description = '';
+                try {
+                    const contentData = await got(item.link);
+                    const $ = cheerio.load(contentData.data);
+                    description = $('#UCAP-CONTENT').html();
+                } catch (error) {
+                    description = '文章已被删除';
+                }
+                item.description = description;
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '信息稿件 - 中国政府网',
+        link: `${link}?${query}`,
+        item: items,
+    };
+};

package/lib/v2/gov/zhengce/wenjian.js

@@ -0,0 +1,47 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { pcodeJiguan } = ctx.params;
+    const link = 'http://sousuo.gov.cn/list.htm';
+    const res = await got(link, {
+        searchParams: {
+            n: ctx.query.limit ? Number(ctx.query.limit) : 20,
+            sort: 'pubtime',
+            t: 'paper',
+            pcodeJiguan: pcodeJiguan ? pcodeJiguan : '',
+        },
+    });
+    const $ = cheerio.load(res.data);
+
+    const list = $('body > div.dataBox > table > tbody > tr')
+        .slice(1)
+        .toArray()
+        .map((elem) => {
+            elem = $(elem);
+            return {
+                title: elem.find('td:nth-child(2) > a').text(),
+                link: elem.find('td:nth-child(2) > a').attr('href'),
+                pubDate: timezone(parseDate(elem.find('td:nth-child(5)').text()), 8),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const contentData = await got(item.link);
+                const $ = cheerio.load(contentData.data);
+                item.description = $('#UCAP-CONTENT').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '最新文件 - 中国政府网',
+        link: res.url,
+        item: items,
+    };
+};

package/lib/v2/gov/zhengce/zhengceku.js

@@ -0,0 +1,22 @@
+const buildData = require('@/utils/common-config');
+
+module.exports = async (ctx) => {
+    const department = ctx.params.department;
+    const link = `http://www.gov.cn/zhengce/zhengceku/${department}/`;
+
+    ctx.state.data = await buildData({
+        link,
+        url: link,
+        title: `%title%`,
+        description: '政府文件库, 当页的所有列表',
+        params: {
+            title: `$('.channel_tab > .noline > a').text().trim() + ' - 政府文件库'`,
+        },
+        item: {
+            item: '.news_box > .list > ul > li:not(.line)',
+            title: `$('h4 > a').text()`,
+            link: `$('h4 > a').attr('href')`,
+            pubDate: `parseDate($('h4 > .date').text().trim())`,
+        },
+    });
+};

package/lib/v2/gov/zhengce/zuixin.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const baseUrl = 'http://www.gov.cn';
+
+module.exports = async (ctx) => {
+    const link = `${baseUrl}/zhengce/zuixin.htm`;
+    const listData = await got(link);
+    const $ = cheerio.load(listData.data);
+
+    const list = $('.news_box .list li:not(.line)')
+        .toArray()
+        .map((elem) => {
+            elem = $(elem);
+            return {
+                title: elem.find('h4 a').text(),
+                link: new URL(elem.find('h4 a').attr('href'), baseUrl).href,
+                pubDate: timezone(parseDate(elem.find('h4 .date').text()), 8),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const contentData = await got(item.link);
+                const $ = cheerio.load(contentData.data);
+                item.description = $('#UCAP-CONTENT').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '最新政策 - 中国政府网',
+        link,
+        item: items,
+    };
+};

package/lib/v2/greasyfork/feedback.js

@@ -0,0 +1,31 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const script = ctx.params.script;
+    const rootUrl = 'https://greasyfork.org';
+    const currentUrl = `${rootUrl}/scripts/${script}/feedback`;
+    const response = await got(currentUrl);
+    const $ = cheerio.load(response.data);
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        description: $('meta[name=description]').attr('content'),
+        item: $('.script-discussion-list .discussion-list-container .discussion-list-item')
+            .get()
+            .map((item) => {
+                item = $(item);
+                const metaItem = item.find('.discussion-meta .discussion-meta-item').eq(0);
+                const discussionTitle = item.find('.discussion-title');
+
+                return {
+                    title: discussionTitle.text().trim(),
+                    author: metaItem.find('a').text(),
+                    pubDate: parseDate(metaItem.find('gf-relative-time').attr('datetime')),
+                    link: rootUrl + discussionTitle.attr('href'),
+                };
+            }),
+    };
+};

package/lib/v2/greasyfork/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/:language/:domain?': ['imlonghao', 'miles170'],
+    '/scripts/:script/feedback': ['miles170'],
+    '/scripts/:script/versions': ['miles170'],
+};

package/lib/v2/greasyfork/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'greasyfork.org': {
+        _name: 'Greasy Fork',
+        '.': [
+            {
+                title: 'User scripts',
+                docs: 'https://docs.rsshub.app/program-update.html#greasy-fork',
+                source: ['/:language', '/:language/scripts/by-site/:domain?'],
+                target: '/greasyfork/:language/:domain?',
+            },
+            {
+                title: 'Feedback',
+                docs: 'https://docs.rsshub.app/program-update.html#greasy-fork',
+                source: '/:language/scripts/:script/feedback',
+                target: '/greasyfork/scripts/:script/feedback',
+            },
+            {
+                title: 'Version history',
+                docs: 'https://docs.rsshub.app/program-update.html#greasy-fork',
+                source: '/:language/scripts/:script/versions',
+                target: '/greasyfork/scripts/:script/versions',
+            },
+        ],
+    },
+};

package/lib/v2/greasyfork/router.js

@@ -0,0 +1,5 @@
+module.exports = (router) => {
+    router.get('/:language/:domain?', require('./scripts'));
+    router.get('/scripts/:script/feedback', require('./feedback'));
+    router.get('/scripts/:script/versions', require('./versions'));
+};

package/lib/{routes → v2}/greasyfork/scripts.js

@@ -7,10 +7,10 @@ module.exports = async (ctx) => {
     const domain = ctx.params.domain;
     const filter_locale = ctx.params.language === 'all' ? 0 : 1;
     const url = domain ? `by-site/${domain}` : '';
-
+    const currentUrl = `https://greasyfork.org/${language}/scripts/${url}`;
     const res = await got({
         method: 'get',
-        url: `https://greasyfork.org/${language}/scripts/${url}`,
+        url: currentUrl,
         searchParams: queryString.stringify({
             filter_locale,
             sort: 'updated',
@@ -21,12 +21,12 @@ module.exports = async (ctx) => {
 
     ctx.state.data = {
         title: $('title').first().text(),
-        link: `https://greasyfork.org/${language}/scripts/${url}`,
+        link: currentUrl,
         description: $('meta[name=description]').attr('content'),
         item:
             list &&
             list
-                .map((index, item) => {
+                .map((_, item) => {
                     item = $(item);
                     return {
                         title: item.find('a').text(),

package/lib/v2/greasyfork/versions.js

@@ -0,0 +1,29 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const script = ctx.params.script;
+    const currentUrl = `https://greasyfork.org/scripts/${script}/versions`;
+    const response = await got(currentUrl);
+    const $ = cheerio.load(response.data);
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        description: $('meta[name=description]').attr('content'),
+        item: $('.history_versions li')
+            .get()
+            .map((item) => {
+                item = $(item);
+                const versionNumberLink = item.find('.version-number a');
+
+                return {
+                    title: versionNumberLink.text(),
+                    description: item.find('.version-changelog').text().trim(),
+                    pubDate: parseDate(item.find('gf-relative-time').attr('datetime')),
+                    link: versionNumberLink.attr('href'),
+                };
+            }),
+    };
+};

package/lib/v2/guanhai/index.js

@@ -0,0 +1,47 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { data: response } = await got('https://www.guanhai.com.cn');
+    const $ = cheerio.load(response);
+
+    const imgBox = $('.img-box');
+    const recommand = {
+        title: imgBox.find('a').first().attr('title'),
+        link: imgBox.find('a').first().attr('href'),
+        pubDate: timezone(parseDate(imgBox.find('time').text()), 8),
+    };
+    const list = $('.pic-summary .title')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: item.find('a').attr('href'),
+                pubDate: timezone(parseDate(item.find('time').text()), 8),
+            };
+        })
+        .concat(recommand);
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+                item.author = $('.source').text();
+                item.description = $('.article-content').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        description: $('head meta[name=description]').text(),
+        image: 'https://www.guanhai.com.cn/favicon.ico',
+        link: 'https://www.guanhai.com.cn',
+        item: items,
+    };
+};

package/lib/v2/guanhai/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '': ['TonyRL'],
+    '/': ['TonyRL'],
+};

package/lib/v2/guanhai/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'guanhai.com.cn': {
+        _name: '观海新闻',
+        '.': [
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-hai-xin-wen',
+                source: ['/'],
+                target: '/guanhai',
+            },
+        ],
+    },
+};

package/lib/v2/guanhai/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/', require('./index'));
+};

package/lib/v2/hackernews/index.js

@@ -8,12 +8,15 @@ module.exports = async (ctx) => {
     const user = ctx.params.user ?? '';
 
     const rootUrl = 'https://news.ycombinator.com';
-    const currentUrl = `${rootUrl}${section === 'index' ? '' : `/${section}`}${user === '' ? '' : '?id=' + user}`;
+    const sectionUrl = section === 'index' ? '' : `/${section}`;
+    let optUrl = user === '' ? '' : '?id=' + user;
 
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
+    if (section === 'over') {
+        optUrl = user === '' ? '?points=100' : '?points=' + user;
+    }
+
+    const currentUrl = `${rootUrl}${sectionUrl}${optUrl}`;
+    const response = await got(currentUrl);
 
     const $ = cheerio.load(response.data);
 
@@ -25,16 +28,18 @@ module.exports = async (ctx) => {
             const item = {};
 
             item.guid = thing.attr('id');
-            item.title = thing.find('.titlelink').text();
+            item.title = thing.find('.titleline').children('a').text();
             item.category = thing.find('.sitestr').text();
             item.author = thing.next().find('.hnuser').text();
             item.pubDate = parseDate(thing.find('.age').attr('title') ?? thing.next().find('.age').attr('title'));
 
             item.link = `${rootUrl}/item?id=${item.guid}`;
-            item.origin = thing.find('.titlelink').attr('href');
+            item.origin = thing.find('.titleline').children('a').attr('href');
             item.onStory = thing.find('.onstory').text().substring(2);
 
             item.comments = thing.next().find('a').last().text().split(' comment')[0];
+            item.upvotes = thing.next().find('.score').text().split(' point')[0];
+
             item.currentComment = thing.find('.comment').text();
             item.guid = type === 'sources' ? item.guid : `${item.guid}${item.comments === 'discuss' ? '' : `-${item.comments}`}`;
 
@@ -86,6 +91,10 @@ module.exports = async (ctx) => {
                     item.description = item.currentComment;
                 }
 
+                if (isNaN(item.comments)) {
+                    item.comments = 0;
+                }
+
                 item.link = type === 'sources' ? item.origin : item.link;
 
                 delete item.origin;

package/lib/v2/hellogithub/index.js

@@ -0,0 +1,102 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const sorts = {
+    hot: '热门',
+    last: '最近',
+};
+
+module.exports = async (ctx) => {
+    const sort = ctx.params.sort ?? 'hot';
+    const id = ctx.params.id ?? '';
+    const limit = ctx.query.limit ? parseInt(ctx.query.limit) : 20;
+
+    const rootUrl = 'https://hellogithub.com';
+    const apiRootUrl = 'https://api.hellogithub.com';
+    const currentUrl = `${rootUrl}/?sort_by=${sort}${id ? `&tid=${id}` : ''}`;
+    const apiUrl = `${apiRootUrl}/v1/?sort_by=${sort}${id ? `&tid=${id}` : ''}&page=1`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let buildId, tag;
+    if (id) {
+        const tagUrl = `${rootUrl}/tags/${id}`;
+
+        const tagResponse = await got({
+            method: 'get',
+            url: tagUrl,
+        });
+
+        const $ = cheerio.load(tagResponse.data);
+
+        tag = $('meta[property="og:title"]').attr('content').split(' ').pop();
+        buildId = tagResponse.data.match(/"buildId":"(.*?)",/)[1];
+    }
+
+    if (!buildId) {
+        const buildResponse = await got({
+            method: 'get',
+            url: rootUrl,
+        });
+
+        buildId = buildResponse.data.match(/"buildId":"(.*?)",/)[1];
+    }
+
+    let items = response.data.data.slice(0, limit).map((item) => ({
+        guid: item.item_id,
+        title: item.title,
+        author: item.author,
+        link: `${rootUrl}/repository/${item.item_id}`,
+        description: item.description,
+        pubDate: parseDate(item.updated_at),
+    }));
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailUrl = `${rootUrl}/_next/data/${buildId}/repository/${item.guid}.json`;
+
+                const detailResponse = await got({
+                    method: 'get',
+                    url: detailUrl,
+                });
+
+                const data = detailResponse.data.pageProps.repo;
+
+                item.title = `${data.name}: ${data.title}`;
+                item.category = [`No.${data.volume_name}`, ...data.tags.map((t) => t.name)];
+                item.description = art(path.join(__dirname, 'templates/description.art'), {
+                    name: data.full_name,
+                    description: data.description,
+                    summary: data.summary,
+                    image: data.image_url,
+                    stars: data.stars ?? data.stars_str,
+                    isChinese: data.has_chinese,
+                    language: data.primary_lang,
+                    isActive: data.is_active,
+                    license: data.license,
+                    isOrganization: data.is_org,
+                    forks: data.forks,
+                    openIssues: data.open_issues,
+                    subscribers: data.subscribers,
+                    homepage: data.homepage,
+                    url: data.url,
+                });
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `HelloGithub - ${sorts[sort]}${tag || ''}项目`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/hellogithub/maintainer.js

@@ -1,5 +1,9 @@
 module.exports = {
-    '/article': ['moke8'],
-    '/ranking/:type?': ['moke8'],
-    '/month': ['moke8'],
+    '/article/:sort?/:id?': ['moke8', 'nczitzk'],
+    '/hot/:id?': ['nczitzk'],
+    '/last/:id?': ['nczitzk'],
+    '/month': ['moke8', 'nczitzk'],
+    '/ranking/:type?': ['moke8', 'nczitzk'],
+    '/report/:type?': ['nczitzk'],
+    '/volume': ['nczitzk'],
 };