rsshub 1.0.0-master.fb2a407 → 1.0.0-master.fc5a2f6

package/lib/v2/gitee/repos/commits.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const { owner, repo } = ctx.params;
+
+    const apiUrl = `https://gitee.com/api/v5/repos/${owner}/${repo}/commits`;
+    const response = await ctx.cache.tryGet(
+        apiUrl,
+        async () =>
+            (
+                await got(apiUrl, {
+                    searchParams: {
+                        access_token: config.gitee.access_token ? config.gitee.access_token : undefined,
+                        per_page: ctx.query.limit ? Number(ctx.query.limit) : 100,
+                        direction: 'desc',
+                    },
+                })
+            ).data
+    );
+
+    const items = response.map((item) => ({
+        title: md.renderInline(item.commit.message),
+        description: md.render(item.commit.message),
+        author: item.author?.login || item.commit.author.name,
+        pubDate: parseDate(item.commit.author.date),
+        guid: item.sha,
+        link: item.html_url,
+    }));
+
+    ctx.state.data = {
+        title: `${owner}/${repo} - 提交`,
+        link: `https://gitee.com/${owner}/${repo}/commits`,
+        item: items,
+    };
+};

package/lib/v2/gitee/repos/events.js

@@ -0,0 +1,66 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const { owner, repo } = ctx.params;
+
+    const apiUrl = `https://gitee.com/api/v5/repos/${owner}/${repo}/events`;
+    const response = await ctx.cache.tryGet(
+        apiUrl,
+        async () =>
+            (
+                await got(apiUrl, {
+                    searchParams: {
+                        access_token: config.gitee.access_token ? config.gitee.access_token : undefined,
+                        limit: ctx.query.limit ? Number(ctx.query.limit) : 100,
+                    },
+                })
+            ).data
+    );
+
+    let items = response.map((item) => ({
+        title: item.type,
+        author: item.actor.login,
+        pubDate: parseDate(item.created_at),
+        guid: item.id,
+        type: item.type,
+        payload: item.payload,
+    }));
+
+    items = items.map((item) => {
+        switch (item.type) {
+            case 'IssueEvent':
+                item.title = item.payload.title;
+                item.description = md.render(item.payload.body);
+                item.link = item.payload.html_url;
+                break;
+            case 'ForkEvent':
+                item.title = `${item.author || item.actor.login} forked ${owner}/${repo}`;
+                item.link = item.payload.html_url;
+                break;
+            case 'StarEvent':
+                item.title = `${item.author || item.actor.login} ${item.payload.action} ${owner}/${repo}`;
+                break;
+            case 'IssueCommentEvent':
+                item.title = item.payload.issue.title;
+                item.description = md.render(item.payload.comment.body);
+                item.link = item.payload.comment.html_url;
+                break;
+            default:
+                break;
+        }
+        delete item.type;
+        delete item.payload;
+        return item;
+    });
+
+    ctx.state.data = {
+        title: `${owner}/${repo} - 仓库动态`,
+        link: `https://gitee.com/${owner}/${repo}`,
+        item: items,
+    };
+};

package/lib/v2/gitee/repos/releases.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const { owner, repo } = ctx.params;
+
+    const response = await got(`https://gitee.com/api/v5/repos/${owner}/${repo}/releases`, {
+        searchParams: {
+            access_token: config.gitee.access_token ? config.gitee.access_token : undefined,
+            per_page: ctx.query.limit ? Number(ctx.query.limit) : 100,
+            direction: 'desc',
+        },
+    });
+
+    const items = response.data.map((item) => ({
+        title: item.tag_name,
+        description: md.render(item.body),
+        author: item.author.login,
+        pubDate: parseDate(item.created_at),
+        guid: item.target_commitish,
+        link: `https://gitee.com/${owner}/${repo}/releases/${item.tag_name}`,
+    }));
+
+    ctx.state.data = {
+        title: `${owner}/${repo} - 发行版`,
+        link: `https://gitee.com/${owner}/${repo}/releases`,
+        item: items,
+    };
+};

package/lib/v2/gitee/router.js

@@ -0,0 +1,6 @@
+module.exports = (router) => {
+    router.get('/commits/:owner/:repo', require('./repos/commits'));
+    router.get('/events/:owner/:repo', require('./repos/events'));
+    router.get('/events/:username', require('./users/events'));
+    router.get('/releases/:owner/:repo', require('./repos/releases'));
+};

package/lib/v2/gitee/users/events.js

@@ -0,0 +1,84 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const { username } = ctx.params;
+
+    const apiUrl = `https://gitee.com/api/v5/users/${username}/events/public`;
+    const response = await ctx.cache.tryGet(
+        apiUrl,
+        async () =>
+            (
+                await got(apiUrl, {
+                    searchParams: {
+                        access_token: config.gitee.access_token ? config.gitee.access_token : undefined,
+                        limit: ctx.query.limit ? Number(ctx.query.limit) : 100,
+                    },
+                })
+            ).data
+    );
+
+    let items = response.map((item) => ({
+        title: item.type,
+        author: item.actor.login,
+        pubDate: parseDate(item.created_at),
+        guid: item.id,
+        type: item.type,
+        repo: item.repo,
+        payload: item.payload,
+    }));
+
+    items = items.map((item) => {
+        switch (item.type) {
+            case 'CommitCommentEvent':
+                item.title = `commented on commit ${item.payload.comment.commit_id.slice(0, 7)} in ${item.payload.repository.full_name}`;
+                item.description = md.render(item.payload.comment.body);
+                item.link = item.payload.comment.html_url;
+                break;
+            case 'CreateEvent':
+                item.title = `${item.payload.ref_type} ${item.payload.ref} created in ${item.repo.full_name}`;
+                break;
+            case 'IssueCommentEvent':
+                item.title = item.payload.issue.title;
+                item.description = md.render(item.payload.comment.body);
+                item.link = item.payload.comment.html_url;
+                break;
+            case 'IssueEvent':
+                item.title = `${item.payload.action} ${item.payload.title}`;
+                item.description = md.render(item.payload.body);
+                item.link = item.payload.html_url;
+                break;
+            case 'ProjectCommentEvent':
+                item.title = `commented on project ${item.repo.full_name}`;
+                item.description = md.render(item.payload.comment.body);
+                item.link = item.payload.comment.html_url;
+                break;
+            case 'PullRequestEvent':
+                item.title = `${item.payload.action} pull request #${item.payload.number} ${item.payload.title} in ${item.repo.full_name}`;
+                item.description = md.render(item.payload.body);
+                item.link = item.payload.html_url;
+                break;
+            case 'PushEvent':
+                item.title = `committed ${item.payload.commits[0].sha.slice(0, 7)} in ${item.repo.full_name}`;
+                item.description = md.render(item.payload.commits[0].message);
+                item.link = `http://gitee.com/${item.repo.full_name}/commit/${item.payload.commits[0].sha}`;
+                break;
+            default:
+                break;
+        }
+        delete item.type;
+        delete item.repo;
+        delete item.payload;
+        return item;
+    });
+
+    ctx.state.data = {
+        title: `${username} - 公开动态`,
+        link: `https://gitee.com/${username}`,
+        item: items,
+    };
+};

package/lib/v2/github/comments.js

@@ -118,5 +118,12 @@ module.exports = async (ctx) => {
         title: `${user}/${repo}: ${typeDict[type].title} #${number} - ${issue.title}`,
         link: issue.html_url,
         item: items,
+        rateLimit: {
+            limit: parseInt(response.headers['x-ratelimit-limit']),
+            remaining: parseInt(response.headers['x-ratelimit-remaining']),
+            reset: parseDate(parseInt(response.headers['x-ratelimit-reset']) * 1000),
+            resoure: response.headers['x-ratelimit-resource'],
+            used: parseInt(response.headers['x-ratelimit-used']),
+        },
     };
 };

package/lib/v2/gitpod/blog.js

@@ -49,4 +49,12 @@ module.exports = async (ctx) => {
         language: 'en-US',
         item: items,
     };
+
+    ctx.state.json = {
+        title: $('title').text(),
+        link: rootUrl + '/blog',
+        description: $('meta[name="description"]').attr('content'),
+        language: 'en-US',
+        item: items,
+    };
 };

package/lib/v2/gov/customs/list.js

@@ -0,0 +1,76 @@
+const cheerio = require('cheerio');
+const { host, puppeteerGet } = require('./utils');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { gchannel = 'paimai' } = ctx.params;
+    let channelName = '';
+    let link = '';
+
+    switch (gchannel) {
+        case 'paimai':
+            channelName = '拍卖信息';
+            link = `${host}/customs/302249/zfxxgk/2799825/2799883/index.html`;
+            break;
+        case 'fagui':
+            channelName = '海关法规';
+            link = `${host}/customs/302249/302266/index.html`;
+            break;
+        default:
+            channelName = '拍卖信息';
+            link = `${host}/customs/302249/zfxxgk/2799825/2799883/index.html`;
+            break;
+    }
+    const response = await ctx.cache.tryGet(
+        link,
+        async () => {
+            const response = await puppeteerGet(link);
+            return response;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const $ = cheerio.load(response);
+
+    const list = $('[class^="conList_ul"] li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: new URL(item.find('a').attr('href'), host).href,
+                date: parseDate(item.find('span').text()),
+            };
+        });
+
+    const out = await Promise.all(
+        list.map((info) =>
+            ctx.cache.tryGet(info.link, async () => {
+                const response = await puppeteerGet(info.link);
+                const $ = cheerio.load(response);
+                let date;
+
+                $('.easysite-news-operation').remove();
+                if (/\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($('.easysite-news-describe').text())) {
+                    date = timezone(parseDate($('.easysite-news-describe').text(), 'YYYY-MM-DD HH:mm'), 8);
+                }
+                const description = $('.easysite-news-peruse').html();
+
+                return {
+                    title: info.title,
+                    link: info.link,
+                    description,
+                    pubDate: date || info.date,
+                };
+            })
+        )
+    );
+    ctx.state.data = {
+        title: `中国海关-${channelName}`,
+        link,
+        item: out,
+    };
+};

package/lib/v2/gov/customs/utils.js

@@ -0,0 +1,17 @@
+const host = 'http://www.customs.gov.cn';
+
+const puppeteerGet = async (url) => {
+    const browser = await require('@/utils/puppeteer')({ stealth: true });
+    const page = await browser.newPage();
+    page.setExtraHTTPHeaders({ referer: host });
+    await page.goto(url);
+    await page.waitForSelector('.pubCon');
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    browser.close();
+    return html;
+};
+
+module.exports = {
+    host,
+    puppeteerGet,
+};

package/lib/v2/gov/hebei/czt.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'gzdt';
+
+    const rootUrl = 'http://czt.hebei.gov.cn';
+    const currentUrl = `${rootUrl}/xwdt/${category}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('td li a[title]')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: `${rootUrl}${/^\.\.\/\.\./.test(item.attr('href')) ? item.attr('href').replace(/^\.\.\/\.\./, '') : `/xwdt/${category}${item.attr('href').replace(/^\./, '')}`}`,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.author = content('meta[name="ContentSource"]').attr('content');
+                item.pubDate = parseDate(content('meta[name="PubDate"]').attr('content'));
+                item.description = content('.TRS_Editor, .content').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/gov/immiau/news.js

@@ -0,0 +1,43 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const url = 'https://immi.homeaffairs.gov.au/_layouts/15/api/Data.aspx/GetNews';
+
+const reqBodyByYear = (year) => ({
+    siteUrl: 'https://www.homeaffairs.gov.au',
+    webUrl: '/News-subsite',
+    filter: {
+        Categories: [],
+        PageNumber: 1,
+        RowLimit: 20,
+        ShowCurrentSiteOnly: false,
+        CurrentSite: 'Immi',
+        Year: year + '',
+    },
+});
+
+const getItemUrl = (id) => `https://immi.homeaffairs.gov.au/news-media/archive/article?itemId=${id}`;
+
+module.exports = async (ctx) => {
+    const res = await got({
+        method: 'post',
+        url,
+        json: reqBodyByYear(new Date().getFullYear()),
+    }).json();
+
+    const list = res.d.data.map((item) => ({
+        title: item.Title,
+        author: item.Source,
+        category: item.Category,
+        description: item.Content,
+        pubDate: parseDate(item.Date),
+        link: getItemUrl(item.Id),
+    }));
+
+    ctx.state.data = {
+        title: 'News - Immigration and Citizenship',
+        link: 'https://immi.homeaffairs.gov.au/news-media/archive',
+        description: 'Australia Government, Department of Home Affairs',
+        item: list,
+    };
+};

package/lib/v2/gov/maintainer.js

@@ -1,23 +1,31 @@
 module.exports = {
-    '/anhui/kjt/:path?': ['nczitzk'],
-    '/beijing/kw/:channel': ['Fatpandac'],
-    '/shenzhen/hrss/szksy/:caty/:page?': ['zlasd'],
-    '/shenzhen/zzb/:caty/:page?': ['zlasd'],
-    '/shanghai/rsj/ksxm': ['Fatpandac'],
+    // ministry
+    '/customs/list/:gchannel?': ['Jeason0228', 'TonyRL'],
     '/fmprc/:category?': ['nicolaszf', 'nczitzk'],
-    '/guangdong/tqyb/tfxtq': ['Fatpandac'],
-    '/guangdong/tqyb/sncsyjxh': ['Fatpandac'],
-    '/huizhou/zwgk/:category?': ['Fatpandac'],
-    '/sichuan/deyang/govpulicinfo/:countyName': ['zytomorrow'],
+    '/immiau/news': ['liu233w'],
+    '/mee/ywdt?': ['liuxsdev'],
     '/mfa/wjdt/:category?': ['nczitzk'],
     '/miit/wjfb/:ministry': ['Fatpandac'],
     '/miit/yjzj': ['Fatpandac'],
     '/miit/zcwj': ['Yoge-Code'],
     '/miit/wjgs': ['Yoge-Code'],
     '/miit/zcjd': ['Yoge-Code'],
-    '/mee/ywdt?': ['liuxsdev'],
+    '/moe/:type': ['Crawler995'],
+    '/nrta/news/:category?': ['yuxinliu-alex'],
     '/pbc/goutongjiaoliu': ['nczitzk'],
+    '/pbc/gzlw': ['Fatpandac'],
     '/pbc/tradeAnnouncement': ['nczitzk'],
     '/pbc/zcyj': ['Fatpandac'],
-    '/pbc/gzlw': ['Fatpandac'],
+    // province
+    '/anhui/kjt/:path?': ['nczitzk'],
+    '/beijing/kw/:channel': ['Fatpandac'],
+    '/hebei/czt/xwdt/:category?': ['nczitzk'],
+    '/shenzhen/hrss/szksy/:caty/:page?': ['zlasd'],
+    '/shenzhen/zzb/:caty/:page?': ['zlasd'],
+    '/shanghai/rsj/ksxm': ['Fatpandac'],
+    '/shanghai/wsjkw/yqtb': ['zcf0508'],
+    '/guangdong/tqyb/tfxtq': ['Fatpandac'],
+    '/guangdong/tqyb/sncsyjxh': ['Fatpandac'],
+    '/huizhou/zwgk/:category?': ['Fatpandac'],
+    '/sichuan/deyang/govpulicinfo/:countyName': ['zytomorrow'],
 };

package/lib/v2/gov/miit/wjgs.js

@@ -1,53 +1,62 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const baseUrl = 'https://www.miit.gov.cn';
+const siteUrl = `${baseUrl}/zwgk/wjgs/index.html`;
 
 module.exports = async (ctx) => {
-    const base_url = 'http://www.miit.gov.cn/n1146295/n7281310/';
-    const response = await got.get(base_url);
+    const response = await got(siteUrl);
     const $ = cheerio.load(response.data);
-    const list = $('.clist_con li').get();
-
-    const ProcessFeed = (data) => {
-        const $ = cheerio.load(data);
-
-        const content = $('p');
-        return content.html();
-    };
-
-    const items = await Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const $a = $('a');
-            let link = $a.attr('href');
-            if (link.startsWith('..')) {
-                link = base_url + link;
-            }
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const response = await got({
-                method: 'get',
-                url: link,
-            });
-
-            const single = {
-                title: $a.text(),
-                description: ProcessFeed(response.data),
-                link,
+    const buildStatic = $('script[parseType=buildstatic]');
+    const requestUrl = buildStatic.attr('url');
+    const queryData = JSON.parse(buildStatic.attr('querydata').replace(/'/g, '"'));
+
+    const { data } = await got(`${baseUrl}${requestUrl}`, {
+        headers: {
+            referer: siteUrl,
+        },
+        searchParams: queryData,
+    });
+    const list = cheerio.load(data.data.html, null, false);
+
+    let items = list('.page-content ul li')
+        .toArray()
+        .map((item) => {
+            item = list(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: new URL(item.find('a').attr('href'), baseUrl).href,
+                pubDate: parseDate(item.find('.fr').text(), 'YYYY-MM-DD'),
             };
-
-            ctx.cache.set(link, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+
+                $('iframe').each((_, e) => {
+                    e = $(e);
+                    if (e.attr('src').startsWith('/')) {
+                        e.attr('src', new URL(e.attr('src'), baseUrl).href);
+                    }
+                });
+                item.author = $('.cinfo')
+                    .text()
+                    .match(/来源：(.*)/)[1];
+                item.pubDate = timezone(parseDate($('#con_time').text(), 'YYYY-MM-DD HH:mm'), +8);
+                item.description = $('.ccontent').html();
+
+                return item;
+            })
+        )
     );
 
     ctx.state.data = {
-        title: '中国工业化和信息部',
-        link: 'http://www.miit.gov.cn',
-        description: '文件公示',
+        title: `${$('head title').text()} - ${$('meta[name=SiteName]').attr('content')}`,
+        link: siteUrl,
         item: items,
     };
 };