rsshub 1.0.0-master.fbcba24 → 1.0.0-master.fc648b5

package/lib/v2/gov/sichuan/deyang/govpulicinfo.js

@@ -4,111 +4,91 @@ const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const path = require('path');
 
-const rootUrl = 'http://xxgk.deyang.gov.cn/xxgkml2020';
-
-// 地区名称对照表
-const nameDict = {
-    德阳市: 'dys',
-    绵竹市: 'mzs',
-    什邡市: 'sfs',
-    旌阳区: 'jyq',
-    罗江区: 'ljx',
-    广汉市: 'ghs',
-    中江县: 'zjx',
-    高新区: 'gxq',
+// 各地区url信息
+const basicInfoDict = {
+    绵竹市: {
+        rootUrl: 'https://www.mz.gov.cn',
+        infoUrl: 'https://www.mz.gov.cn/info/iList.jsp?tm_id=1805&cur_page=',
+    },
 };
 
-const getInstitutionId = async (ctx, county) => {
-    const url = `${rootUrl}/ptlj.jsp?regionName=${county}`;
-
-    return await ctx.cache.tryGet(`${county}InstitutionId`, async () => {
-        const response = await got.get(url);
+const getInfoUrlList = async (countyName) => {
+    const url = basicInfoDict[countyName].infoUrl;
+    const infoUrlList = [];
+    const expectToday = new Date();
+    for (let pageNum = 1; ; pageNum++) {
+        let response;
+        try {
+            // eslint-disable-next-line no-await-in-loop
+            response = await got(`${url}${pageNum}`);
+        } catch (err) {
+            // eslint-disable-next-line no-await-in-loop
+            response = await got(`${url}${pageNum}`, { dnsLookupIpVersion: 'ipv6' });
+        }
         const $ = cheerio.load(response.data);
-        const dataList = $('#details_content > div > div > div > div:nth-child(4) > ul > li > a');
-        const _tmp = {};
-        for (let i = 0; i < dataList.length; i++) {
-            _tmp[$(dataList[i]).html()] = parseInt($(dataList[i]).attr('href').split('deptId=')[1]);
+        const infoList = $('#list_content > ul > li');
+        // 判断当前页是否有数据
+        if (infoList.length !== 0) {
+            for (let infoIdx = 0; infoIdx < infoList.length; infoIdx++) {
+                const date = parseDate($('span', infoList[infoIdx]).html());
+                // 判断当前信息日期是否是今日
+                if (date.getUTCFullYear() === expectToday.getUTCFullYear() && date.getUTCMonth() === expectToday.getUTCMonth() && date.getUTCDate() === expectToday.getUTCDate() - 1) {
+                    infoUrlList.push(`${basicInfoDict[countyName].rootUrl}${$('a', infoList[infoIdx]).attr('href')}`);
+                } else {
+                    return infoUrlList;
+                }
+            }
+        } else {
+            return infoUrlList;
         }
-        return _tmp;
-    });
-};
-
-const getInfoUrlList = async (county, institutionId) => {
-    const url = `${rootUrl}/gklist_iframe.jsp?deptId=${institutionId}&regionName=${county}&pageSize=15`;
-    const response = await got.get(url);
-    const $ = cheerio.load(response.data);
-    const pageNum = parseInt($('#list_content > div > span:nth-child(3)').html().match(/\d*/g)[1]);
-    const pageUrlList = [];
-
-    // 此处建议限制最大pageNum,最大5页,单页15条
-    for (let i = 1; i <= Math.min(pageNum, 5); i++) {
-        pageUrlList.push(`${rootUrl}/gklist_iframe.jsp?deptId=${institutionId}&regionName=${county}&pageSize=15$pageIndex=${i}`);
-    }
-    const _tmpList = await Promise.all(
-        pageUrlList.map(async (url) => {
-            const response = await got.get(url);
-            const $ = cheerio.load(response.data);
-            const InfoList = $('#list_content > ul > li > a');
-            return InfoList.map((item) => `${rootUrl}/${$(InfoList[item]).attr('href')}`);
-        })
-    );
-    let infoUrlList = [];
-    for (let i = 0; i < _tmpList.length; i++) {
-        infoUrlList = infoUrlList.concat(_tmpList[i].toArray());
     }
-    return infoUrlList;
 };
 
-const getInfoContent = async (ctx, url) => {
-    const infoId = url.split('id=')[1].split('&type')[0];
-    return await ctx.cache.tryGet(`govPublicInfo${infoId}`, async () => {
-        const response = await got.get(url);
+// 获取信息正文内容
+const getInfoContent = async (ctx, url, countyName) =>
+    await ctx.cache.tryGet(url, async () => {
+        let response;
+        try {
+            response = await got(url);
+        } catch (err) {
+            response = await got(url, { dnsLookupIpVersion: 'ipv6' });
+        }
         const $ = cheerio.load(response.data);
-        const fileNodes = $('#symbol > div:nth-child(3) > div > a');
+        const fileNodes = $('#symbol > div:nth-child(4) > div > span > a');
         const fileList = [];
         for (let i = 0; i < fileNodes.length; i++) {
             fileList.push({
                 name: $(fileNodes[i]).text(),
-                url: `${rootUrl}/${$(fileNodes[i]).attr('href')}`,
+                url: `${basicInfoDict[countyName].rootUrl}/${$(fileNodes[i]).attr('href')}`,
             });
         }
-        const rawDate = $('#symbol > div:nth-child(1) > div:nth-child(3)').text().split('\n')[1].trim();
+        const rawDate = $('#symbol > div:nth-child(1) > div:nth-child(3)').text().split('：')[1].trim();
         return {
-            title: $('#headline').text(),
-            id: infoId,
-            infoNum: $('#symbol > div:nth-child(1) > div:nth-child(2) > span').text().split('\n')[1].trim(),
+            title: $('#main').text().trim(),
+            id: $('#symbol > div:nth-child(1) > div:nth-child(1)').text().split('：')[1].trim(),
+            infoNum: $('#symbol > div:nth-child(1) > div:nth-child(2) > span').text().split('：')[1].trim(),
             pubDate: parseDate(rawDate),
             date: rawDate,
-            keyWord: $('#symbol > div:nth-child(2) > div:nth-child(1)').text().slice(8),
-            source: $('#symbol > div:nth-child(2) > div:nth-child(3)').text().slice(5),
-            content: $('#details_content > div.content').html(),
+            keyWord: $('#symbol > div:nth-child(2) > div:nth-child(3)').text().split('：')[1].trim(),
+            source: $('#symbol > div:nth-child(2) > div:nth-child(2)').text().split('：')[1].trim(),
+            content: $('#container > div.ewb-white > div.ewb-article-detail').html(),
             file: fileList,
             link: url,
         };
     });
-};
 
 module.exports = async (ctx) => {
     const countyName = ctx.params.countyName;
-    const county = nameDict[countyName];
-    const institutionName = ctx.params.institutionName || '';
-    let institutionId = 0;
-    if (institutionName) {
-        const institutionDict = await getInstitutionId(ctx, county);
-        institutionId = institutionDict[institutionName];
-    }
-
-    const infoUrlList = await getInfoUrlList(county, institutionId);
-    const items = await Promise.all(infoUrlList.map(async (item) => await getInfoContent(ctx, item)));
+    const infoUrlList = await getInfoUrlList(countyName);
+    const items = await Promise.all(infoUrlList.map(async (item) => await getInfoContent(ctx, item, countyName)));
 
     ctx.state.data = {
-        title: `政府公开信息 - ${countyName} ${institutionName}`,
-        link: `${rootUrl}/gklist_iframe.jsp?deptId=${institutionId}&regionName=${county}`,
+        title: `政府公开信息 - ${countyName}`,
+        link: basicInfoDict[countyName].infoUrl,
         item: items.map((item) => ({
             title: item.title,
             description: art(path.resolve(__dirname, './templates/govPublicInfo.art'), { item }),
             link: item.link,
-            guid: item.id,
             pubDate: item.pubDate,
         })),
     };

package/lib/v2/gov/sichuan/deyang/templates/govPublicInfo.art

@@ -33,6 +33,7 @@
     </tbody>
 </table>
 
+<hr/>
 <div>
     {{@ item.content}}
 </div>

package/lib/v2/gov/xuzhou/hrss.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? '';
+
+    const rootUrl = 'http://hrss.xz.gov.cn';
+    const currentUrl = `${rootUrl}${category ? `/001/${category}/subPage.html` : ''}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = (category ? $('.module-items a') : $('.bdl[data-target="1"]').eq(1).find('a')).toArray().map((item) => {
+        item = $(item);
+
+        const link = item.attr('href');
+
+        return {
+            title: item.attr('title'),
+            link: `${/^http/.test(link) ? '' : rootUrl}${item.attr('href')}`,
+        };
+    });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description = content('#Zoom, .mian-cont, .ewb-article-info, #UCAP-CONTENT').html();
+                item.pubDate = timezone(parseDate(content('meta[name="PubDate"]').attr('content')), +8);
+                item.category = content('meta[name="Keywords"]').attr('content')?.split(' ');
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `徐州市人力资源和社会保障局 - ${category ? $('.wb-tree-items.current').text() : '通知公告'}`,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/guancha/headline.js

@@ -0,0 +1,51 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://www.guancha.cn';
+    const currentUrl = `${rootUrl}/GuanChaZheTouTiao/list_1.shtml`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.headline-list li .content-headline h3 a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.text(),
+                description: item.parent().next().html(),
+                link: `${rootUrl}${item.attr('href').replace(/\.shtml$/, '_s.shtml')}`,
+                pubDate: timezone(parseDate(item.parents('div').first().find('span').text()), +8),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.description += content('.all-txt').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '观察者网 - 头条',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/guancha/index.js

@@ -0,0 +1,159 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const { parseRelativeDate } = require('@/utils/parse-date');
+
+const config = {
+    review: {
+        title: '评论 & 研究',
+        query: '.module-news-main',
+    },
+    story: {
+        title: '要闻',
+        query: '.img-List',
+    },
+    fengwen: {
+        title: '风闻',
+        query: '.fengwen-list',
+    },
+    redian: {
+        title: '热点',
+    },
+    gundong: {
+        title: '滚动',
+    },
+    all: {
+        title: '全部',
+    },
+    home: {
+        title: '首页',
+    },
+    others: {
+        title: '热点 & 滚动',
+    },
+};
+
+module.exports = async (ctx) => {
+    const total = 10;
+    const category = ctx.params.category ?? 'all';
+    const rootUrl = 'https://www.guancha.cn';
+
+    let newsList = [],
+        redianList = [],
+        gundongList = [];
+
+    // 'review', 'story' and 'fengwen' come from homepage.
+
+    if (category === 'review' || category === 'story' || category === 'fengwen' || category === 'all' || category === 'home') {
+        const response = await got({
+            method: 'get',
+            url: rootUrl,
+        });
+
+        const $ = cheerio.load(response.data);
+
+        const fetchPost = (slice) =>
+            slice
+                .find('h4.module-title a')
+                .toArray()
+
+                // Filter some blank links which lead to no contents but 'https://user.guancha.cn'.
+
+                .filter((item) => $(item).attr('href') !== 'https://user.guancha.cn')
+                .map((item) => {
+                    item = $(item);
+
+                    const link = item.attr('href');
+
+                    return {
+                        title: item.text(),
+                        link: `${link.indexOf('http') === 0 ? '' : rootUrl}${link.replace(/\.shtml/, '_s.shtml')}`,
+                    };
+                });
+
+        if (category === 'all' || category === 'home') {
+            newsList = fetchPost($(config.review.query))
+                .slice(0, total / 3)
+                .concat(fetchPost($(config.story.query)).slice(0, total / 3), fetchPost($(config.fengwen.query)).slice(0, total / 3));
+        } else {
+            newsList = fetchPost($(config[category].query)).slice(0, total);
+        }
+    }
+
+    // 'redian' and 'gundong' come from api.
+
+    if (category === 'redian' || category === 'all' || category === 'others') {
+        const response = await got({
+            method: 'get',
+            url: `${rootUrl}/api/redian.htm`,
+        });
+
+        redianList = response.data.items
+            .map((item) => ({
+                title: item.TITLE,
+                link: `${rootUrl}${item.HTTP_URL.replace(/\.shtml/, '_s.shtml')}`,
+            }))
+            .slice(0, category === 'all' ? total / 3 : total);
+    }
+
+    if (category === 'gundong' || category === 'all' || category === 'others') {
+        const response = await got({
+            method: 'get',
+            url: `${rootUrl}/api/gundong.htm`,
+        });
+
+        gundongList = response.data.items
+            .map((item) => ({
+                title: item.TITLE,
+                link: `${rootUrl}${item.HTTP_URL.replace(/\.shtml/, '_s.shtml')}`,
+            }))
+            .slice(0, category === 'all' ? total / 3 : total);
+    }
+
+    const items = await Promise.all(
+        newsList.concat(redianList, gundongList).map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                let detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                // Some links to posts in 'fengwen' must be redirected in order to fetch content.
+                // eg. https://www.guancha.cn/politics/2020_10_23_569021.shtml
+                //  => https://user.guancha.cn/main/content?id=399176
+
+                const jumpMatch = detailResponse.data.match(/user.guancha.cn\/main\/content\?id=(.*)";/);
+
+                if (jumpMatch !== null) {
+                    item.link = `https://user.guancha.cn/main/content?id=${jumpMatch[1]}&page=0`;
+                    detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+                }
+
+                const content = cheerio.load(detailResponse.data);
+
+                const dateMatch = detailResponse.data.match(/"pubDate": "(.*)"/);
+                if (dateMatch === null) {
+                    // PubDates of posts in 'fengwen' are in an informal format.
+
+                    item.pubDate = parseRelativeDate(content('.time1').text());
+                } else {
+                    item.pubDate = timezone(parseDate(dateMatch[1]), +8);
+                }
+
+                item.description = content('.all-txt').html() || content('.article-txt-content').html();
+                item.author = content('.author-intro p a').text() || content('.article-content div div h4 a').text() || content('.editor-intro p a').text() || content('.left-main > div.time.fix > span').eq(2).text();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `观察者网 - ${config[category].title}`,
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/guancha/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/:category?': ['nczitzk', 'Jeason0228'],
+    '/headline': ['nczitzk'],
+    '/member/:category?': ['nczitzk'],
+    '/personalpage/:uid': ['Jeason0228'],
+    '/topic/:id/:order?': ['occupy5', 'nczitzk'],
+};

package/lib/v2/guancha/member.js

@@ -0,0 +1,86 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+
+const titles = {
+    recommend: '精选',
+    books: '观书堂',
+    courses: '在线课',
+    huodongs: '观学院',
+};
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category ?? 'recommend';
+
+    const rootUrl = 'https://member.guancha.cn';
+    const apiUrl = `${rootUrl}/zaixianke/home`;
+
+    const response = await got({
+        method: 'get',
+        url: apiUrl,
+    });
+
+    let items;
+
+    switch (category) {
+        case 'books':
+            items = response.data.data.books.map((item) => ({
+                title: item.title,
+                link: `${rootUrl}/guanshutang/summary.html?id=${item.id}&page=0`,
+                description: `<img src="${item.cover}"><p>[${item.audio_time}] ${item.desc_short}</p>`,
+                pubDate: new Date(parseInt(item.cover.split('/').pop().substr(0, 10)) * 1000).toUTCString(),
+            }));
+            break;
+
+        case 'courses':
+            items = response.data.data.courses.data.map((item) => {
+                let description = '',
+                    pubDate = new Date(0);
+
+                for (const i of item.items) {
+                    const newPubDate = new Date(i.publish_time);
+                    pubDate = pubDate > newPubDate ? pubDate : newPubDate;
+                    description += `<a href="${rootUrl}/zaixianke/content.html?id=${i.id}">${i.title}</a><br>`;
+                }
+
+                return {
+                    title: item.name,
+                    link: `${rootUrl}/zaixianke/summary.html?id=${item.id}`,
+                    author: item.author_name,
+                    description: `<img src="${item.cover}"><p>${item.desc_short}</p><br>${description}`,
+                    pubDate: pubDate.toUTCString(),
+                };
+            });
+            break;
+
+        default:
+            items = response.data.data[category].map((item) => {
+                let timeArray = item.media_time && item.media_time.trim().split(/\D+/, 3);
+                timeArray = timeArray && timeArray.filter((item) => item !== '');
+                let itunes_duration;
+                if (timeArray) {
+                    itunes_duration = 0;
+                    itunes_duration += timeArray.length >= 1 ? parseInt(timeArray.slice(-1)) : 0;
+                    itunes_duration += timeArray.length >= 2 ? parseInt(timeArray.slice(-2)) * 60 : 0;
+                    itunes_duration += timeArray.length >= 3 ? parseInt(timeArray.slice(-3)) * 60 * 60 : 0;
+                }
+                return {
+                    title: item.title,
+                    link: item.jump_url,
+                    author: item.author_name,
+                    description: `<img src="${item.big_pic}"><p>${item.summary}</p>`,
+                    enclosure_url: item.media_url,
+                    enclosure_length: item.media_size,
+                    itunes_duration,
+                    enclosure_type: 'audio/mpeg',
+                    pubDate: isNaN(+item.created_at) ? timezone(parseDate(item.created_at), +8) : parseDate(item.created_at * 1000),
+                };
+            });
+    }
+
+    ctx.state.data = {
+        title: `观学院 - ${titles[category]}`,
+        link: `${rootUrl}/index.html`,
+        item: items,
+    };
+};

package/lib/v2/guancha/radar.js

@@ -0,0 +1,49 @@
+module.exports = {
+    'guancha.cn': {
+        _name: '观察者网',
+        '.': [
+            {
+                title: '头条',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-tou-tiao',
+                source: ['/GuanChaZheTouTiao', '/'],
+                target: '/guancha/headline',
+            },
+            {
+                title: '首页',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-shou-ye',
+                source: ['/'],
+                target: '/guancha/:category?',
+            },
+            {
+                title: '观学院',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-guan-xue-yuan',
+                source: ['/'],
+                target: '/guancha/:category?',
+            },
+        ],
+        app: [
+            {
+                title: '个人主页文章',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-ge-ren-zhu-ye-wen-zhang',
+                source: ['/user/get-published-list', '/'],
+                target: (params, url) => `/guancha/personalpage/${new URL(url).searchParams.get('uid')}`,
+            },
+        ],
+        member: [
+            {
+                title: '观学院',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-guan-xue-yuan',
+                source: ['/'],
+                target: '/guancha/member/recommend',
+            },
+        ],
+        user: [
+            {
+                title: '风闻话题',
+                docs: 'https://docs.rsshub.app/new-media.html#guan-cha-zhe-wang-feng-wen-hua-ti',
+                source: ['/topic/post-list', '/'],
+                target: (params, url) => `/guancha/topic/${new URL(url).searchParams.get('topic_id')}/${new URL(url).searchParams.get('order')}`,
+            },
+        ],
+    },
+};

package/lib/v2/guancha/router.js

@@ -0,0 +1,7 @@
+module.exports = function (router) {
+    router.get('/headline', require('./headline'));
+    router.get('/member/:category?', require('./member'));
+    router.get('/personalpage/:uid', require('./personalpage'));
+    router.get('/topic/:id/:order?', require('./topic'));
+    router.get('/:category?', require('./index'));
+};

package/lib/v2/guancha/topic.js

@@ -0,0 +1,56 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseRelativeDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id ?? '0';
+    const order = ctx.params.order ?? '1';
+
+    const rootUrl = 'https://user.guancha.cn';
+    const currentUrl = `${rootUrl}/${id === '0' ? 'main/index-list.json?' : 'topic/post-list?topic_id=' + id}&page=1&order=${order}`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.list-item h4 a, ul.home li .list-item h4 a')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+
+            return {
+                title: item.text(),
+                link: `${rootUrl}${item.attr('href')}&page=0`,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = cheerio.load(detailResponse.data);
+
+                item.pubDate = parseRelativeDate(content('.time1').text());
+                item.author = content('.user-main h4 a').first().text();
+                item.description = content('.article-txt-content').html();
+
+                return item;
+            })
+        )
+    );
+
+    $('h1.title span').remove();
+
+    ctx.state.data = {
+        title: `观察者网 - ${id === '0' ? '风闻' : $('h1.title').text()}`,
+        link: currentUrl,
+        item: items,
+    };
+};