rsshub 1.0.0-master.ff1cc11 → 1.0.0-master.ff97759

package/lib/v2/gov/router.js

@@ -0,0 +1,28 @@
+module.exports = function (router) {
+    // ministry
+    router.get('/fmprc/:category?', require('./mfa/wjdt'));
+    router.get('/mee/ywdt/:category?', require('./mee/ywdt'));
+    router.get('/mfa/wjdt/:category?', require('./mfa/wjdt'));
+    router.get('/miit/wjfb/:ministry', require('./miit/wjfb'));
+    router.get('/miit/yjzj', require('./miit/yjzj'));
+    router.get('/miit/zcwj', require('./miit/zcwj'));
+    router.get('/miit/wjgs', require('./miit/wjgs'));
+    router.get('/miit/zcjd', require('./miit/zcjd'));
+    router.get('/moe/:type', require('./moe/moe'));
+    router.get('/nrta/news/:category?', require('./nrta/news'));
+    router.get('/pbc/goutongjiaoliu', require('./pbc/goutongjiaoliu'));
+    router.get('/pbc/gzlw', require('./pbc/gzlw'));
+    router.get('/pbc/tradeAnnouncement', require('./pbc/tradeAnnouncement'));
+    router.get('/pbc/zcyj', require('./pbc/zcyj'));
+    // province
+    router.get(/anhui\/kjt\/([\w\d/-]+)?/, require('./anhui/kjt'));
+    router.get('/beijing/kw/:channel', require('./beijing/kw/index'));
+    router.get('/shenzhen/hrss/szksy/:caty/:page?', require('./shenzhen/hrss/szksy/index'));
+    router.get('/shenzhen/zzb/:caty/:page?', require('./shenzhen/zzb/index'));
+    router.get('/shanghai/rsj/ksxm', require('./shanghai/rsj/ksxm'));
+    router.get('/shanghai/wsjkw/yqtb', require('./shanghai/wsjkw/yqtb'));
+    router.get('/guangdong/tqyb/tfxtq', require('./guangdong/tqyb/tfxtq'));
+    router.get('/guangdong/tqyb/sncsyjxh', require('./guangdong/tqyb/sncsyjxh'));
+    router.get('/huizhou/zwgk/:category?', require('./huizhou/zwgk/index'));
+    router.get('/sichuan/deyang/govpulicinfo/:countyName', require('./sichuan/deyang/govpulicinfo'));
+};

package/lib/v2/gov/shanghai/rsj/ksxm.js

@@ -0,0 +1,39 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const iconv = require('iconv-lite');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+const rootUrl = 'http://www.rsj.sh.gov.cn';
+
+module.exports = async (ctx) => {
+    const url = `${rootUrl}/ksyzc/wangz/kwaplist_300.jsp`;
+
+    const response = await got({
+        method: 'get',
+        url,
+        responseType: 'buffer',
+    });
+    const dataHtml = iconv.decode(response.data, 'gbk');
+    const $ = cheerio.load(dataHtml);
+
+    const items = $('kwap')
+        .map((_, item) => ({
+            title: $(item).find('kaosxmmc').text(),
+            link: `http://www.rsj.sh.gov.cn/ksyzc/index801.jsp`,
+            description: art(path.join(__dirname, './templates/ksxm.art'), {
+                name: $(item).find('kaosxmmc').text(),
+                type: $(item).find('kaoslb_dmfy').text(),
+                date: $(item).find('kaosrq').text(),
+                registrationDeadline: $(item).find('baomksrq_A300').text(),
+            }),
+            guid: `${$(item).find('kaosrq').text()}${$(item).find('kaosxmmc').text()}`,
+        }))
+        .get();
+
+    ctx.state.data = {
+        title: '上海市职业能力考试院 - 考试项目',
+        link: url,
+        item: items,
+    };
+};

package/lib/v2/gov/shanghai/rsj/templates/ksxm.art

@@ -0,0 +1,7 @@
+<text>考试项目名称：{{ name }} </text>
+<br>
+<text>考试类别：{{ type }} </text>
+<br>
+<text>考试日期：{{ date }} </text>
+<br>
+<text>报名起止日期：{{ registrationDeadline }} </text>

package/lib/v2/gov/shanghai/wsjkw/yqtb/index.js

@@ -0,0 +1,33 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const url = `https://wsjkw.sh.gov.cn/yqtb/index.html`;
+
+    const res = await got.get(url);
+    const $ = cheerio.load(res.data);
+    const list = $('.uli16.nowrapli.list-date  li');
+    ctx.state.data = {
+        title: '疫情通报-上海卫健委',
+        link: url,
+        item:
+            list &&
+            list
+                .map((index, item) => {
+                    item = $(item);
+                    const title = item.find('a').text();
+                    const address = item.find('a').attr('href');
+                    const host = `https://wsjkw.sh.gov.cn`;
+                    const pubDate = parseDate(item.find('span').text(), 'YYYY-MM-DD');
+                    return {
+                        title,
+                        description: title,
+                        pubDate,
+                        link: host + address,
+                        guid: host + address,
+                    };
+                })
+                .get(),
+    };
+};

package/lib/v2/gov/shenzhen/hrss/szksy/index.js

@@ -0,0 +1,41 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const rootURL = 'http://hrss.sz.gov.cn/';
+
+module.exports = async (ctx) => {
+    const categoryID = ctx.params.caty;
+    const page = ctx.params.page ?? '1';
+
+    const pageParam = parseInt(page) > 1 ? `_${page}` : '';
+    const pagePath = `/szksy/zwgk/${categoryID}/index${pageParam}.html`;
+
+    const currentURL = new URL(pagePath, rootURL); // do not use deprecated 'url.resolve'
+    const response = await got({ method: 'get', url: currentURL });
+    if (response.statusCode !== 200) {
+        throw new Error(response.statusMessage);
+    }
+
+    const $ = cheerio.load(response.data);
+    const title = $('.zx_rm_tit span').text().trim();
+    const list = $('.zx_ml_list ul li')
+        .slice(1)
+        .map((_, item) => {
+            const tag = $(item).find('div.list_name a');
+            const tag2 = $(item).find('span:eq(1)');
+            return {
+                title: tag.text().trim(),
+                link: tag.attr('href'),
+                pubDate: timezone(parseDate(tag2.text().trim(), 'YYYY/MM/DD'), 0),
+            };
+        })
+        .get();
+
+    ctx.state.data = {
+        title: '深圳市考试院 - ' + title,
+        link: currentURL.href,
+        item: list,
+    };
+};

package/lib/v2/gov/shenzhen/zzb/index.js

@@ -0,0 +1,40 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const rootURL = 'http://www.zzb.sz.gov.cn/';
+
+module.exports = async (ctx) => {
+    const categoryID = ctx.params.caty;
+    const page = ctx.params.page ?? '1';
+
+    const pageParam = parseInt(page) > 1 ? `_${page}` : '';
+    const pagePath = `/${categoryID}/index${pageParam}.html`;
+
+    const currentURL = new URL(pagePath, rootURL); // do not use deprecated 'url.resolve'
+    const response = await got({ method: 'get', url: currentURL });
+    if (response.statusCode !== 200) {
+        throw new Error(response.statusMessage);
+    }
+
+    const $ = cheerio.load(response.data);
+    const title = $('#Title').text().trim();
+    const list = $('#List tbody tr td table tbody tr td[width="96%"]')
+        .map((_, item) => {
+            const tag = $(item).find('font a');
+            const tag2 = $(item).find('font[size="2px"]');
+            return {
+                title: tag.text(),
+                link: tag.attr('href'),
+                pubDate: timezone(parseDate(tag2.text().trim(), 'YYYY/MM/DD'), 0),
+            };
+        })
+        .get();
+
+    ctx.state.data = {
+        title: '深圳组工在线 - ' + title,
+        link: currentURL.href,
+        item: list,
+    };
+};

package/lib/v2/gov/sichuan/deyang/govpulicinfo.js

@@ -0,0 +1,95 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { art } = require('@/utils/render');
+const path = require('path');
+
+// 各地区url信息
+const basicInfoDict = {
+    绵竹市: {
+        rootUrl: 'https://www.mz.gov.cn',
+        infoUrl: 'https://www.mz.gov.cn/info/iList.jsp?tm_id=1805&cur_page=',
+    },
+};
+
+const getInfoUrlList = async (countyName) => {
+    const url = basicInfoDict[countyName].infoUrl;
+    const infoUrlList = [];
+    const expectToday = new Date();
+    for (let pageNum = 1; ; pageNum++) {
+        let response;
+        try {
+            // eslint-disable-next-line no-await-in-loop
+            response = await got(`${url}${pageNum}`);
+        } catch (err) {
+            // eslint-disable-next-line no-await-in-loop
+            response = await got(`${url}${pageNum}`, { dnsLookupIpVersion: 'ipv6' });
+        }
+        const $ = cheerio.load(response.data);
+        const infoList = $('#list_content > ul > li');
+        // 判断当前页是否有数据
+        if (infoList.length !== 0) {
+            for (let infoIdx = 0; infoIdx < infoList.length; infoIdx++) {
+                const date = parseDate($('span', infoList[infoIdx]).html());
+                // 判断当前信息日期是否是今日
+                if (date.getUTCFullYear() === expectToday.getUTCFullYear() && date.getUTCMonth() === expectToday.getUTCMonth() && date.getUTCDate() === expectToday.getUTCDate() - 1) {
+                    infoUrlList.push(`${basicInfoDict[countyName].rootUrl}${$('a', infoList[infoIdx]).attr('href')}`);
+                } else {
+                    return infoUrlList;
+                }
+            }
+        } else {
+            return infoUrlList;
+        }
+    }
+};
+
+// 获取信息正文内容
+const getInfoContent = async (ctx, url, countyName) =>
+    await ctx.cache.tryGet(url, async () => {
+        let response;
+        try {
+            response = await got(url);
+        } catch (err) {
+            response = await got(url, { dnsLookupIpVersion: 'ipv6' });
+        }
+        const $ = cheerio.load(response.data);
+        const fileNodes = $('#symbol > div:nth-child(4) > div > span > a');
+        const fileList = [];
+        for (let i = 0; i < fileNodes.length; i++) {
+            fileList.push({
+                name: $(fileNodes[i]).text(),
+                url: `${basicInfoDict[countyName].rootUrl}/${$(fileNodes[i]).attr('href')}`,
+            });
+        }
+        const rawDate = $('#symbol > div:nth-child(1) > div:nth-child(3)').text().split('：')[1].trim();
+        return {
+            title: $('#main').text().trim(),
+            id: $('#symbol > div:nth-child(1) > div:nth-child(1)').text().split('：')[1].trim(),
+            infoNum: $('#symbol > div:nth-child(1) > div:nth-child(2) > span').text().split('：')[1].trim(),
+            pubDate: parseDate(rawDate),
+            date: rawDate,
+            keyWord: $('#symbol > div:nth-child(2) > div:nth-child(3)').text().split('：')[1].trim(),
+            source: $('#symbol > div:nth-child(2) > div:nth-child(2)').text().split('：')[1].trim(),
+            content: $('#container > div.ewb-white > div.ewb-article-detail').html(),
+            file: fileList,
+            link: url,
+        };
+    });
+
+module.exports = async (ctx) => {
+    const countyName = ctx.params.countyName;
+    const infoUrlList = await getInfoUrlList(countyName);
+    const items = await Promise.all(infoUrlList.map(async (item) => await getInfoContent(ctx, item, countyName)));
+
+    ctx.state.data = {
+        title: `政府公开信息 - ${countyName}`,
+        link: basicInfoDict[countyName].infoUrl,
+        item: items.map((item) => ({
+            title: item.title,
+            description: art(path.resolve(__dirname, './templates/govPublicInfo.art'), { item }),
+            link: item.link,
+            pubDate: item.pubDate,
+        })),
+    };
+};

package/lib/v2/gov/sichuan/deyang/templates/govPublicInfo.art

@@ -0,0 +1,42 @@
+<table border="1" cellpadding="2" cellspacing="0" align="center">
+    <tbody>
+            <tr>
+                <td>索引号</td>
+                <td>{{item.id}}</td>
+            </tr>
+            <tr>
+                <td>文号</td>
+                <td>{{item.infoNum}}</td>
+            </tr>
+            <tr>
+                <td>发文日期</td>
+                <td>{{item.date}}</td>
+            </tr>
+            <tr>
+                <td>关键词</td>
+                <td>{{item.keyWord}}</td>
+            </tr>
+            <tr>
+                <td>信息来源</td>
+                <td>{{item.source}}</td>
+            </tr>
+            {{if item.file.length}}
+                <tr>
+                    <td>附件</td>
+                    <td>
+                        {{each item.file}}
+                            <a href="{{$value.url}}">{{$value.name}}</a><br>
+                        {{/each}}
+                    </td>
+                </tr>
+            {{/if}}
+    </tbody>
+</table>
+
+<hr/>
+<div>
+    {{@ item.content}}
+</div>
+
+
+

package/lib/v2/guancha/headline.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.guancha.cn';
+    const currentUrl = `${rootUrl}/GuanChaZheTouTiao/list_1.shtml`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.headline-list li .content-headline h3 a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                description: item.parent().next().html(),
+                link: `${rootUrl}${item.attr('href').replace(/\.shtml$/, '_s.shtml')}`,
+                pubDate: timezone(parseDate(item.parents('div').first().find('span').text()), +8),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description += content('.all-txt').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '观察者网 - 头条',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/guancha/index.js

@@ -0,0 +1,159 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { parseRelativeDate } = require('@/utils/parse-date');
+
+const config = {
+    review: {
+        title: '评论 & 研究',
+        query: '.module-news-main',
+    },
+    story: {
+        title: '要闻',
+        query: '.img-List',
+    },
+    fengwen: {
+        title: '风闻',
+        query: '.fengwen-list',
+    },
+    redian: {
+        title: '热点',
+    },
+    gundong: {
+        title: '滚动',
+    },
+    all: {
+        title: '全部',
+    },
+    home: {
+        title: '首页',
+    },
+    others: {
+        title: '热点 & 滚动',
+    },
+};
+
+module.exports = async (ctx) => {
+    const total = 10;
+    const category = ctx.params.category ?? 'all';
+    const rootUrl = 'https://www.guancha.cn';
+
+    let newsList = [],
+        redianList = [],
+        gundongList = [];
+
+    // 'review', 'story' and 'fengwen' come from homepage.
+
+    if (category === 'review' || category === 'story' || category === 'fengwen' || category === 'all' || category === 'home') {
+        const response = await got({
+            method: 'get',
+            url: rootUrl,
+        });
+
+        const $ = cheerio.load(response.data);
+
+        const fetchPost = (slice) =>
+            slice
+                .find('h4.module-title a')
+                .toArray()
+
+                // Filter some blank links which lead to no contents but 'https://user.guancha.cn'.
+
+                .filter((item) => $(item).attr('href') !== 'https://user.guancha.cn')
+                .map((item) => {
+                    item = $(item);
+
+                    const link = item.attr('href');
+
+                    return {
+                        title: item.text(),
+                        link: `${link.indexOf('http') === 0 ? '' : rootUrl}${link.replace(/\.shtml/, '_s.shtml')}`,
+                    };
+                });
+
+        if (category === 'all' || category === 'home') {
+            newsList = fetchPost($(config.review.query))
+                .slice(0, total / 3)
+                .concat(fetchPost($(config.story.query)).slice(0, total / 3), fetchPost($(config.fengwen.query)).slice(0, total / 3));
+        } else {
+            newsList = fetchPost($(config[category].query)).slice(0, total);
+        }
+    }
+
+    // 'redian' and 'gundong' come from api.
+
+    if (category === 'redian' || category === 'all' || category === 'others') {
+        const response = await got({
+            method: 'get',
+            url: `${rootUrl}/api/redian.htm`,
+        });
+
+        redianList = response.data.items
+            .map((item) => ({
+                title: item.TITLE,
+                link: `${rootUrl}${item.HTTP_URL.replace(/\.shtml/, '_s.shtml')}`,
+            }))
+            .slice(0, category === 'all' ? total / 3 : total);
+    }
+
+    if (category === 'gundong' || category === 'all' || category === 'others') {
+        const response = await got({
+            method: 'get',
+            url: `${rootUrl}/api/gundong.htm`,
+        });
+
+        gundongList = response.data.items
+            .map((item) => ({
+                title: item.TITLE,
+                link: `${rootUrl}${item.HTTP_URL.replace(/\.shtml/, '_s.shtml')}`,
+            }))
+            .slice(0, category === 'all' ? total / 3 : total);
+    }
+
+    const items = await Promise.all(
+        newsList.concat(redianList, gundongList).map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                let detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                // Some links to posts in 'fengwen' must be redirected in order to fetch content.
+                // eg. https://www.guancha.cn/politics/2020_10_23_569021.shtml
+                //  => https://user.guancha.cn/main/content?id=399176
+
+                const jumpMatch = detailResponse.data.match(/user.guancha.cn\/main\/content\?id=(.*)";/);
+
+                if (jumpMatch !== null) {
+                    item.link = `https://user.guancha.cn/main/content?id=${jumpMatch[1]}&page=0`;
+                    detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+                }
+
+                const content = cheerio.load(detailResponse.data);
+
+                const dateMatch = detailResponse.data.match(/"pubDate": "(.*)"/);
+                if (dateMatch === null) {
+                    // PubDates of posts in 'fengwen' are in an informal format.
+
+                    item.pubDate = parseRelativeDate(content('.time1').text());
+                } else {
+                    item.pubDate = timezone(parseDate(dateMatch[1]), +8);
+                }
+
+                item.description = content('.all-txt').html() || content('.article-txt-content').html();
+                item.author = content('.author-intro p a').text() || content('.article-content div div h4 a').text() || content('.editor-intro p a').text() || content('.left-main > div.time.fix > span').eq(2).text();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `观察者网 - ${config[category].title}`,
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/guancha/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/:category?': ['nczitzk', 'Jeason0228'],
+    '/headline': ['nczitzk'],
+    '/member/:category?': ['nczitzk'],
+    '/personalpage/:uid': ['Jeason0228'],
+    '/topic/:id/:order?': ['occupy5', 'nczitzk'],
+};

package/lib/v2/guancha/member.js

@@ -0,0 +1,86 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+const titles = {
+    recommend: '精选',
+    books: '观书堂',
+    courses: '在线课',
+    huodongs: '观学院',
+};
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'recommend';
+
+    const rootUrl = 'https://member.guancha.cn';
+    const apiUrl = `${rootUrl}/zaixianke/home`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let items;
+
+    switch (category) {
+        case 'books':
+            items = response.data.data.books.map((item) => ({
+                title: item.title,
+                link: `${rootUrl}/guanshutang/summary.html?id=${item.id}&page=0`,
+                description: `<img src="${item.cover}"><p>[${item.audio_time}] ${item.desc_short}</p>`,
+                pubDate: new Date(parseInt(item.cover.split('/').pop().substr(0, 10)) * 1000).toUTCString(),
+            }));
+            break;
+
+        case 'courses':
+            items = response.data.data.courses.data.map((item) => {
+                let description = '',
+                    pubDate = new Date(0);
+
+                for (const i of item.items) {
+                    const newPubDate = new Date(i.publish_time);
+                    pubDate = pubDate > newPubDate ? pubDate : newPubDate;
+                    description += `<a href="${rootUrl}/zaixianke/content.html?id=${i.id}">${i.title}</a><br>`;
+                }
+
+                return {
+                    title: item.name,
+                    link: `${rootUrl}/zaixianke/summary.html?id=${item.id}`,
+                    author: item.author_name,
+                    description: `<img src="${item.cover}"><p>${item.desc_short}</p><br>${description}`,
+                    pubDate: pubDate.toUTCString(),
+                };
+            });
+            break;
+
+        default:
+            items = response.data.data[category].map((item) => {
+                let timeArray = item.media_time && item.media_time.trim().split(/\D+/, 3);
+                timeArray = timeArray && timeArray.filter((item) => item !== '');
+                let itunes_duration;
+                if (timeArray) {
+                    itunes_duration = 0;
+                    itunes_duration += timeArray.length >= 1 ? parseInt(timeArray.slice(-1)) : 0;
+                    itunes_duration += timeArray.length >= 2 ? parseInt(timeArray.slice(-2)) * 60 : 0;
+                    itunes_duration += timeArray.length >= 3 ? parseInt(timeArray.slice(-3)) * 60 * 60 : 0;
+                }
+                return {
+                    title: item.title,
+                    link: item.jump_url,
+                    author: item.author_name,
+                    description: `<img src="${item.big_pic}"><p>${item.summary}</p>`,
+                    enclosure_url: item.media_url,
+                    enclosure_length: item.media_size,
+                    itunes_duration,
+                    enclosure_type: 'audio/mpeg',
+                    pubDate: isNaN(+item.created_at) ? timezone(parseDate(item.created_at), +8) : parseDate(item.created_at * 1000),
+                };
+            });
+    }
+
+    ctx.state.data = {
+        title: `观学院 - ${titles[category]}`,
+        link: `${rootUrl}/index.html`,
+        item: items,
+    };
+};