rsshub 1.0.0-master.f7c0247 → 1.0.0-master.f7c4a93

package/lib/v2/gitee/repos/events.js

@@ -0,0 +1,66 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const { owner, repo } = ctx.params;
+
+    const apiUrl = `https://gitee.com/api/v5/repos/${owner}/${repo}/events`;
+    const response = await ctx.cache.tryGet(
+        apiUrl,
+        async () =>
+            (
+                await got(apiUrl, {
+                    searchParams: {
+                        access_token: config.gitee.access_token ? config.gitee.access_token : undefined,
+                        limit: ctx.query.limit ? Number(ctx.query.limit) : 100,
+                    },
+                })
+            ).data
+    );
+
+    let items = response.map((item) => ({
+        title: item.type,
+        author: item.actor.login,
+        pubDate: parseDate(item.created_at),
+        guid: item.id,
+        type: item.type,
+        payload: item.payload,
+    }));
+
+    items = items.map((item) => {
+        switch (item.type) {
+            case 'IssueEvent':
+                item.title = item.payload.title;
+                item.description = md.render(item.payload.body);
+                item.link = item.payload.html_url;
+                break;
+            case 'ForkEvent':
+                item.title = `${item.author || item.actor.login} forked ${owner}/${repo}`;
+                item.link = item.payload.html_url;
+                break;
+            case 'StarEvent':
+                item.title = `${item.author || item.actor.login} ${item.payload.action} ${owner}/${repo}`;
+                break;
+            case 'IssueCommentEvent':
+                item.title = item.payload.issue.title;
+                item.description = md.render(item.payload.comment.body);
+                item.link = item.payload.comment.html_url;
+                break;
+            default:
+                break;
+        }
+        delete item.type;
+        delete item.payload;
+        return item;
+    });
+
+    ctx.state.data = {
+        title: `${owner}/${repo} - 仓库动态`,
+        link: `https://gitee.com/${owner}/${repo}`,
+        item: items,
+    };
+};

package/lib/v2/gitee/repos/releases.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const { owner, repo } = ctx.params;
+
+    const response = await got(`https://gitee.com/api/v5/repos/${owner}/${repo}/releases`, {
+        searchParams: {
+            access_token: config.gitee.access_token ? config.gitee.access_token : undefined,
+            per_page: ctx.query.limit ? Number(ctx.query.limit) : 100,
+            direction: 'desc',
+        },
+    });
+
+    const items = response.data.map((item) => ({
+        title: item.tag_name,
+        description: md.render(item.body),
+        author: item.author.login,
+        pubDate: parseDate(item.created_at),
+        guid: item.target_commitish,
+        link: `https://gitee.com/${owner}/${repo}/releases/${item.tag_name}`,
+    }));
+
+    ctx.state.data = {
+        title: `${owner}/${repo} - 发行版`,
+        link: `https://gitee.com/${owner}/${repo}/releases`,
+        item: items,
+    };
+};

package/lib/v2/gitee/router.js

@@ -0,0 +1,6 @@
+module.exports = (router) => {
+    router.get('/commits/:owner/:repo', require('./repos/commits'));
+    router.get('/events/:owner/:repo', require('./repos/events'));
+    router.get('/events/:username', require('./users/events'));
+    router.get('/releases/:owner/:repo', require('./repos/releases'));
+};

package/lib/v2/gitee/users/events.js

@@ -0,0 +1,84 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const config = require('@/config').value;
+const md = require('markdown-it')({
+    html: true,
+});
+
+module.exports = async (ctx) => {
+    const { username } = ctx.params;
+
+    const apiUrl = `https://gitee.com/api/v5/users/${username}/events/public`;
+    const response = await ctx.cache.tryGet(
+        apiUrl,
+        async () =>
+            (
+                await got(apiUrl, {
+                    searchParams: {
+                        access_token: config.gitee.access_token ? config.gitee.access_token : undefined,
+                        limit: ctx.query.limit ? Number(ctx.query.limit) : 100,
+                    },
+                })
+            ).data
+    );
+
+    let items = response.map((item) => ({
+        title: item.type,
+        author: item.actor.login,
+        pubDate: parseDate(item.created_at),
+        guid: item.id,
+        type: item.type,
+        repo: item.repo,
+        payload: item.payload,
+    }));
+
+    items = items.map((item) => {
+        switch (item.type) {
+            case 'CommitCommentEvent':
+                item.title = `commented on commit ${item.payload.comment.commit_id.slice(0, 7)} in ${item.payload.repository.full_name}`;
+                item.description = md.render(item.payload.comment.body);
+                item.link = item.payload.comment.html_url;
+                break;
+            case 'CreateEvent':
+                item.title = `${item.payload.ref_type} ${item.payload.ref} created in ${item.repo.full_name}`;
+                break;
+            case 'IssueCommentEvent':
+                item.title = item.payload.issue.title;
+                item.description = md.render(item.payload.comment.body);
+                item.link = item.payload.comment.html_url;
+                break;
+            case 'IssueEvent':
+                item.title = `${item.payload.action} ${item.payload.title}`;
+                item.description = md.render(item.payload.body);
+                item.link = item.payload.html_url;
+                break;
+            case 'ProjectCommentEvent':
+                item.title = `commented on project ${item.repo.full_name}`;
+                item.description = md.render(item.payload.comment.body);
+                item.link = item.payload.comment.html_url;
+                break;
+            case 'PullRequestEvent':
+                item.title = `${item.payload.action} pull request #${item.payload.number} ${item.payload.title} in ${item.repo.full_name}`;
+                item.description = md.render(item.payload.body);
+                item.link = item.payload.html_url;
+                break;
+            case 'PushEvent':
+                item.title = `committed ${item.payload.commits[0].sha.slice(0, 7)} in ${item.repo.full_name}`;
+                item.description = md.render(item.payload.commits[0].message);
+                item.link = `http://gitee.com/${item.repo.full_name}/commit/${item.payload.commits[0].sha}`;
+                break;
+            default:
+                break;
+        }
+        delete item.type;
+        delete item.repo;
+        delete item.payload;
+        return item;
+    });
+
+    ctx.state.data = {
+        title: `${username} - 公开动态`,
+        link: `https://gitee.com/${username}`,
+        item: items,
+    };
+};

package/lib/v2/gov/beijing/jw/tzgg.js

@@ -0,0 +1,58 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'http://jw.beijing.gov.cn';
+    const currentUrl = `${rootUrl}/tzgg`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.col-md a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            const link = item.attr('href');
+
+            return {
+                title: item.text(),
+                link: /^http/.test(link) ? link : `${rootUrl}${link.replace(/^\./, '/tzgg')}`,
+                pubDate: parseDate(item.parent().find('span').text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                const pubDate = content('meta[name="PubDate"]').attr('content');
+
+                item.author = content('meta[name="ContentSource"]').attr('content');
+                item.pubDate = pubDate ? timezone(parseDate(content('meta[name="PubDate"]').attr('content')), +8) : item.pubDate;
+                item.description = content('.TRS_UEDITOR').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '北京市教育委员会 - 通知公告',
+        link: currentUrl,
+        item: items,
+        description: $('meta[name="ColumnDescription"]').attr('content'),
+    };
+};

package/lib/v2/gov/customs/list.js

@@ -0,0 +1,80 @@
+const cheerio = require('cheerio');
+const { host, puppeteerGet } = require('./utils');
+const config = require('@/config').value;
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const { gchannel = 'paimai' } = ctx.params;
+    let channelName = '';
+    let link = '';
+
+    switch (gchannel) {
+        case 'paimai':
+            channelName = '拍卖信息';
+            link = `${host}/customs/302249/zfxxgk/2799825/2799883/index.html`;
+            break;
+        case 'fagui':
+            channelName = '海关法规';
+            link = `${host}/customs/302249/302266/index.html`;
+            break;
+        default:
+            channelName = '拍卖信息';
+            link = `${host}/customs/302249/zfxxgk/2799825/2799883/index.html`;
+            break;
+    }
+
+    const browser = await require('@/utils/puppeteer')();
+
+    const list = await ctx.cache.tryGet(
+        link,
+        async () => {
+            const response = await puppeteerGet(link, browser);
+            const $ = cheerio.load(response);
+            const list = $('[class^="conList_ul"] li')
+                .toArray()
+                .map((item) => {
+                    item = $(item);
+                    return {
+                        title: item.find('a').attr('title'),
+                        link: new URL(item.find('a').attr('href'), host).href,
+                        date: parseDate(item.find('span').text()),
+                    };
+                });
+            return list;
+        },
+        config.cache.routeExpire,
+        false
+    );
+
+    const out = await Promise.all(
+        list.map((info) =>
+            ctx.cache.tryGet(info.link, async () => {
+                const response = await puppeteerGet(info.link, browser);
+                const $ = cheerio.load(response);
+                let date;
+
+                $('.easysite-news-operation').remove();
+                if (/\d{4}-\d{2}-\d{2} \d{2}:\d{2}/.test($('.easysite-news-describe').text())) {
+                    date = timezone(parseDate($('.easysite-news-describe').text(), 'YYYY-MM-DD HH:mm'), 8);
+                }
+                const description = $('.easysite-news-peruse').html();
+
+                return {
+                    title: info.title,
+                    link: info.link,
+                    description,
+                    pubDate: date || info.date,
+                };
+            })
+        )
+    );
+
+    browser.close();
+
+    ctx.state.data = {
+        title: `中国海关-${channelName}`,
+        link,
+        item: out,
+    };
+};

package/lib/v2/gov/customs/utils.js

@@ -0,0 +1,21 @@
+const host = 'http://www.customs.gov.cn';
+
+const puppeteerGet = async (url, browser) => {
+    const page = await browser.newPage();
+    await page.setExtraHTTPHeaders({ referer: host });
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(url, {
+        waitUntil: 'domcontentloaded',
+    });
+    await page.waitForSelector('.pubCon');
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    return html;
+};
+
+module.exports = {
+    host,
+    puppeteerGet,
+};

package/lib/v2/gov/maintainer.js

@@ -1,4 +1,6 @@
 module.exports = {
+    // ministry
+    '/customs/list/:gchannel?': ['Jeason0228', 'TonyRL'],
     '/fmprc/:category?': ['nicolaszf', 'nczitzk'],
     '/immiau/news': ['liu233w'],
     '/mee/ywdt?': ['liuxsdev'],
@@ -14,8 +16,9 @@ module.exports = {
     '/pbc/gzlw': ['Fatpandac'],
     '/pbc/tradeAnnouncement': ['nczitzk'],
     '/pbc/zcyj': ['Fatpandac'],
-
+    // province
     '/anhui/kjt/:path?': ['nczitzk'],
+    '/beijing/jw/tzgg': ['nczitzk'],
     '/beijing/kw/:channel': ['Fatpandac'],
     '/hebei/czt/xwdt/:category?': ['nczitzk'],
     '/shenzhen/hrss/szksy/:caty/:page?': ['zlasd'],

package/lib/v2/gov/miit/wjgs.js

@@ -1,53 +1,62 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const baseUrl = 'https://www.miit.gov.cn';
+const siteUrl = `${baseUrl}/zwgk/wjgs/index.html`;
 
 module.exports = async (ctx) => {
-    const base_url = 'http://www.miit.gov.cn/n1146295/n7281310/';
-    const response = await got.get(base_url);
+    const response = await got(siteUrl);
     const $ = cheerio.load(response.data);
-    const list = $('.clist_con li').get();
-
-    const ProcessFeed = (data) => {
-        const $ = cheerio.load(data);
-
-        const content = $('p');
-        return content.html();
-    };
-
-    const items = await Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const $a = $('a');
-            let link = $a.attr('href');
-            if (link.startsWith('..')) {
-                link = base_url + link;
-            }
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const response = await got({
-                method: 'get',
-                url: link,
-            });
-
-            const single = {
-                title: $a.text(),
-                description: ProcessFeed(response.data),
-                link,
+    const buildStatic = $('script[parseType=buildstatic]');
+    const requestUrl = buildStatic.attr('url');
+    const queryData = JSON.parse(buildStatic.attr('querydata').replace(/'/g, '"'));
+
+    const { data } = await got(`${baseUrl}${requestUrl}`, {
+        headers: {
+            referer: siteUrl,
+        },
+        searchParams: queryData,
+    });
+    const list = cheerio.load(data.data.html, null, false);
+
+    let items = list('.page-content ul li')
+        .toArray()
+        .map((item) => {
+            item = list(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: new URL(item.find('a').attr('href'), baseUrl).href,
+                pubDate: parseDate(item.find('.fr').text(), 'YYYY-MM-DD'),
             };
-
-            ctx.cache.set(link, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+
+                $('iframe').each((_, e) => {
+                    e = $(e);
+                    if (e.attr('src').startsWith('/')) {
+                        e.attr('src', new URL(e.attr('src'), baseUrl).href);
+                    }
+                });
+                item.author = $('.cinfo')
+                    .text()
+                    .match(/来源：(.*)/)[1];
+                item.pubDate = timezone(parseDate($('#con_time').text(), 'YYYY-MM-DD HH:mm'), +8);
+                item.description = $('.ccontent').html();
+
+                return item;
+            })
+        )
     );
 
     ctx.state.data = {
-        title: '中国工业化和信息部',
-        link: 'http://www.miit.gov.cn',
-        description: '文件公示',
+        title: `${$('head title').text()} - ${$('meta[name=SiteName]').attr('content')}`,
+        link: siteUrl,
         item: items,
     };
 };

package/lib/v2/gov/miit/zcjd.js

@@ -1,53 +1,62 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const baseUrl = 'https://www.miit.gov.cn';
+const siteUrl = `${baseUrl}/zwgk/zcjd/index.html`;
 
 module.exports = async (ctx) => {
-    const base_url = 'http://www.miit.gov.cn/n1146295/n7281315/';
-    const response = await got.get(base_url);
+    const response = await got(siteUrl);
     const $ = cheerio.load(response.data);
-    const list = $('.clist_con li').get();
-
-    const ProcessFeed = (data) => {
-        const $ = cheerio.load(data);
-
-        const content = $('p');
-        return content.html();
-    };
-
-    const items = await Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const $a = $('a');
-            let link = $a.attr('href');
-            if (link.startsWith('..')) {
-                link = base_url + link;
-            }
-
-            const cache = await ctx.cache.get(link);
-            if (cache) {
-                return Promise.resolve(JSON.parse(cache));
-            }
-
-            const response = await got({
-                method: 'get',
-                url: link,
-            });
-
-            const single = {
-                title: $a.text(),
-                description: ProcessFeed(response.data),
-                link,
+    const buildStatic = $('script[parseType=buildstatic]');
+    const requestUrl = buildStatic.attr('url');
+    const queryData = JSON.parse(buildStatic.attr('querydata').replace(/'/g, '"'));
+
+    const { data } = await got(`${baseUrl}${requestUrl}`, {
+        headers: {
+            referer: siteUrl,
+        },
+        searchParams: queryData,
+    });
+    const list = cheerio.load(data.data.html, null, false);
+
+    let items = list('.page-content ul li')
+        .toArray()
+        .map((item) => {
+            item = list(item);
+            return {
+                title: item.find('a').attr('title'),
+                link: new URL(item.find('a').attr('href'), baseUrl).href,
+                pubDate: parseDate(item.find('.fr').text(), 'YYYY-MM-DD'),
             };
-
-            ctx.cache.set(link, JSON.stringify(single));
-            return Promise.resolve(single);
-        })
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data } = await got(item.link);
+                const $ = cheerio.load(data);
+
+                $('iframe').each((_, e) => {
+                    e = $(e);
+                    if (e.attr('src').startsWith('/')) {
+                        e.attr('src', new URL(e.attr('src'), baseUrl).href);
+                    }
+                });
+                item.author = $('.cinfo')
+                    .text()
+                    .match(/来源：(.*)/)[1];
+                item.pubDate = timezone(parseDate($('#con_time').text(), 'YYYY-MM-DD HH:mm'), +8);
+                item.description = $('.ccontent').html();
+
+                return item;
+            })
+        )
     );
 
     ctx.state.data = {
-        title: '中国工业化和信息部',
-        link: 'http://www.miit.gov.cn',
-        description: '政策解读',
+        title: `${$('head title').text()} - ${$('meta[name=SiteName]').attr('content')}`,
+        link: siteUrl,
         item: items,
     };
 };

package/lib/v2/gov/pbc/goutongjiaoliu.js

@@ -5,22 +5,25 @@ const timezone = require('@/utils/timezone');
 module.exports = async (ctx) => {
     const link = 'http://www.pbc.gov.cn/goutongjiaoliu/113456/113469/index.html';
 
-    const browser = await require('@/utils/puppeteer')();
+    const browser = await require('@/utils/puppeteer')({ stealth: true });
     const page = await browser.newPage();
-    await page.goto(link);
-    const html = await page.evaluate(
-        () =>
-            // eslint-disable-next-line
-            document.querySelector('body').innerHTML
-    );
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+    });
+    await page.goto(link, {
+        waitUntil: 'domcontentloaded',
+    });
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+
     const $ = cheerio.load(html);
     const list = $('font.newslist_style')
         .map((_, item) => {
             item = $(item);
             const a = item.find('a[title]');
             return {
-                title: a.text(),
-                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn'),
+                title: a.attr('title'),
+                link: new URL(a.attr('href'), 'http://www.pbc.gov.cn').href,
             };
         })
         .get();
@@ -29,12 +32,14 @@ module.exports = async (ctx) => {
         list.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailPage = await browser.newPage();
-                await detailPage.goto(item.link);
-                const detailHtml = await detailPage.evaluate(
-                    () =>
-                        // eslint-disable-next-line
-                        document.querySelector('body').innerHTML
-                );
+                await detailPage.setRequestInterception(true);
+                detailPage.on('request', (request) => {
+                    request.resourceType() === 'document' || request.resourceType() === 'script' ? request.continue() : request.abort();
+                });
+                await detailPage.goto(item.link, {
+                    waitUntil: 'domcontentloaded',
+                });
+                const detailHtml = await detailPage.evaluate(() => document.documentElement.innerHTML);
                 const content = cheerio.load(detailHtml);
                 item.description = content('#zoom').html();
                 item.pubDate = timezone(parseDate(content('.hui12').eq(5).text()), +8);