rsshub 1.0.0-master.fe50f9c → 1.0.0-master.fe560d2

package/lib/v2/good/index.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://good.news';
+
+    const response = await got(rootUrl);
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.content-structure')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item
+                    .find('h4')
+                    .text()
+                    .replace(/^\s+\d{1,2}\./, ''),
+                link: item.parent().attr('href'),
+                pubDate: parseDate(item.find('.content-left').text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('img').each((_, img) => {
+                    if (img.attribs['data-src']) {
+                        img.attribs.src = img.attribs['data-src'].replace('-cdsb.onlycompress', '');
+                        delete img.attribs['data-src'];
+                    }
+                });
+
+                item.description = content('[contentscore]').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'good.news - 今日要闻',
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/good/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['nczitzk'],
+};

package/lib/v2/good/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'good.news': {
+        _name: 'Good.news',
+        '.': [
+            {
+                title: '今日要闻',
+                docs: 'https://docs.rsshub.app/new-media.html#good-news-jin-ri-yao-wen',
+                source: ['/'],
+                target: '/good',
+            },
+        ],
+    },
+};

package/lib/v2/good/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index'));
+};

package/lib/v2/gov/beijing/bphc/index.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const mapping = {
+    '/project': {
+        title: '项目介绍',
+        list: '/front/index/project/page',
+        detail: '/front/index/project',
+        link: '/#/communityDetail?id=',
+    },
+    '/announcement': {
+        title: '通知公告',
+        list: '/front/announcement/page',
+        detail: '/front/announcement/',
+        link: '/#/announcementList/detail?id=',
+    },
+};
+
+module.exports = async (ctx) => {
+    const rootUrl = 'http://gycp.bphc.com.cn:21000';
+    const defaultPath = 'announcement';
+
+    const pathname = ctx.path.replace(/(^\/beijing\/bphc|\/$)/g, '');
+    const key = pathname === '' ? defaultPath : pathname.endsWith('/') ? pathname.slice(0, -1) : pathname;
+    const obj = mapping[key];
+    const currentUrl = `${rootUrl}${obj.list}`;
+
+    const listResp = await got(currentUrl).json();
+    const list = listResp.data?.records ?? [];
+    const items = await Promise.all(
+        list.map((item) => {
+            const detail = `${rootUrl}${obj.detail}/${item.id}`;
+            return ctx.cache.tryGet(detail, async () => {
+                const detailResponse = await got(detail).json();
+                const description = (detailResponse.data?.content || detailResponse.data?.introduction) ?? '';
+                const single = {
+                    title: item.title || item.fullName,
+                    author: description.match(/来源：(.*?)</)?.[1].trim() ?? item.operator,
+                    link: `${rootUrl}${obj.link}${item.id}`,
+                    description,
+                    pubDate: timezone(parseDate(item.createTime), +8),
+                };
+                return single;
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: obj.title,
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/gov/ccdi/index.js

@@ -0,0 +1,18 @@
+const { rootUrl, parseNewsList, parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const defaultPath = '/yaowenn/';
+
+    let pathname = ctx.path.replace(/(^\/ccdi|\/$)/g, '');
+    pathname = pathname === '' ? defaultPath : pathname.endsWith('/') ? pathname : pathname + '/';
+    const currentUrl = `${rootUrl}${pathname}`;
+
+    const { list, title } = await parseNewsList(currentUrl, '.list_news_dl li', ctx);
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx)));
+
+    ctx.state.data = {
+        title,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/gov/ccdi/utils.js

@@ -0,0 +1,94 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+
+const { CookieJar, Cookie } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+const owner = '中央纪委国家监委网站';
+const rootUrl = 'https://www.ccdi.gov.cn';
+const regex = /(?<key>[A-Z_]+)=(?<value>(?:.*?(?=; max-age)|[a-fA-F0-9]+))/gm;
+
+const parseCookie = async (body) => {
+    let m;
+    const cookies = [];
+    while ((m = regex.exec(body)) !== null) {
+        // This is necessary to avoid infinite loops with zero-width matches
+        if (m.index === regex.lastIndex) {
+            regex.lastIndex++;
+        }
+        const { key, value } = m.groups;
+        cookies.push(new Cookie({ key, value }));
+    }
+    await Promise.all(cookies.map((c) => cookieJar.setCookie(c, rootUrl)));
+};
+
+const parseNewsList = async (url, selector, ctx) => {
+    const response = await got(url, { cookieJar });
+    const data = response.data;
+    await parseCookie(data);
+
+    const $ = cheerio.load(data);
+    const list = $(selector)
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').first().text().trim(),
+                link: new URL(item.find('a').first().attr('href'), url).href,
+                pubDate: parseDate(item.find('span').text(), 'YYYY-MM-DD'),
+            };
+        });
+    const title = $('.other_Location')
+        .text()
+        .replace(/(.+)首页/, owner);
+    return { list, title };
+};
+
+const changeTrCookie = async () => {
+    const cookies = await cookieJar.getCookies(rootUrl);
+    const c = cookies.find((c) => c.key === 'HOY_TR');
+    if (c) {
+        const value = c.value;
+        const tr_array = value.split(',');
+        const csr = tr_array[0];
+        const cnv = tr_array[1].split('');
+        const otr = tr_array[2].split('');
+        otr[0] = csr.charAt(parseInt(cnv[0], 16));
+        const nc = new Cookie({ key: 'HOY_TR', value: csr + ',' + cnv.join('') + ',' + otr.join('') + ',0' });
+        await cookieJar.setCookie(nc, rootUrl);
+    }
+};
+
+const parseArticle = async (item, ctx) => {
+    await changeTrCookie();
+    return await ctx.cache.tryGet(item.link, async () => {
+        const response = await got(item.link, { cookieJar });
+        const data = response.data;
+        await parseCookie(data);
+
+        const $ = cheerio.load(data);
+        const title = $('.daty, .source-box').text().trim();
+        item.author = title.match(/来源：(.*)发布时间/s)?.[1].trim() ?? owner;
+        item.pubDate = timezone(parseDate(title.match(/发布时间：(.*)分享/s)?.[1].trim() ?? item.pubDate), +8);
+
+        // Change the img src from relative to absolute for a better compatibility
+        $('.content, .bom-box')
+            .find('img')
+            .each((_, el) => {
+                $(el).attr('src', new URL($(el).attr('src'), item.link).href);
+                // oldsrc is causing freshrss imageproxy not to work correctly
+                $(el).removeAttr('oldsrc').removeAttr('alt');
+            });
+        item.description = $('.content, .bom-box').html();
+        return item;
+    });
+};
+
+module.exports = {
+    rootUrl,
+    parseNewsList,
+    parseArticle,
+};

package/lib/v2/gov/customs/list.js

@@ -24,7 +24,7 @@ module.exports = async (ctx) => {
             break;
     }
 
-    const browser = await require('@/utils/puppeteer')();
+    const browser = await require('@/utils/puppeteer')({ stealth: true });
 
     const list = await ctx.cache.tryGet(
         link,

package/lib/v2/gov/dianbai/dianbai.js

@@ -0,0 +1,18 @@
+const { gdgov } = require('../general/general');
+
+module.exports = async (ctx) => {
+    const info = {
+        defaultPath: 'zwgk/zcjd/',
+        list_element: '.news_list li a',
+        title_element: '.content_title',
+        title_match: '(.*)',
+        description_element: '#zoomcon',
+        author_element: undefined,
+        author_match: undefined,
+        authorisme: '茂名市电白区人民政府网',
+        pubDate_element: 'publishtime',
+        pubDate_match: '(.*)',
+        pubDate_format: undefined,
+    };
+    await gdgov(info, ctx);
+};

package/lib/v2/gov/gaozhou/gaozhou.js

@@ -0,0 +1,19 @@
+const { gdgov } = require('../general/general');
+
+module.exports = async (ctx) => {
+    const info = {
+        defaultPath: 'zcjd/',
+        list_element: '.newslist li a',
+        list_include: 'site',
+        title_element: '.head',
+        title_match: '(.*)',
+        description_element: '.contener',
+        author_element: undefined,
+        author_match: undefined,
+        authorisme: '高州市人民政府网',
+        pubDate_element: '.time span:nth-child(1)',
+        pubDate_match: '发布时间：(.*)',
+        pubDate_format: undefined,
+    };
+    await gdgov(info, ctx);
+};

package/lib/v2/gov/general/general.js

@@ -0,0 +1,250 @@
+// 来人拯救一下啊( >﹏<。)
+// 待做功能：
+// 1. 传入和处理
+//         [] index.php?c=category&id=12
+// 2. 处理其他网站链接
+//         [] 其他政府网站
+// 3. 添加视频内容
+//         [] 示例1: http://www.dianbai.gov.cn/ywdt/dbyw/content/post_1091433.html
+// 4. 处理网站功能
+//        [] hdjlpt 互动交流
+
+// 使用方法
+// const { gdgov } = require('../general/general');
+//
+// module.exports = async (ctx) => {
+//     const info = {
+//         pathstartat: 1,                                              // 网站放到子目录时使用，默认为0。如 www.yunfu.gov.cn/yfsrmzf/jcxxgk/zcfg/zcjd 为一层子目录则使用 1。
+//         defaultPath: 'zwgk/zcjd/',                                   // 默认路径。假设网址是 a.b.gov.cn/c/d/ 则输入 c/d/。访问 gov/b/a/ 时使用默认路径，访问 gov/b/a/c/d/ 则为指定路径。
+//         name_element: 'SiteName, ColumnName',                        // 网站名，默认使用网页标题。
+//         name_match_type: 'meta',                                     // 网站名类型。可选 meta（从 <meta> 中获取 content 里面的内容）、name（name_element 就是网站名）、element（选择元素从中获取文本）。
+//         name_match: '(.*)',                                          // 网站名类型为 element 时使用的匹配内容。
+//         name_join: '—',                                             // 网站名为 meta 或 element 时，如有多个选择器找到内容，将这里的字符串放到这些内容之间。
+//         list_element: '.news_list li a',                             // 页面列表中选择具体到 a。element 都是填写 CSS 选择器。
+//         list_include: 'site',                                        // 筛选列表中的页面，会生成选择器添加到 list_element 中。可选 all（包含全部）、site（仅限本站）。
+//         title_element: '.content_title',                             // 正文的标题，默认使用 <meta name="ArticleTitle" content="*">
+//         title_match: '(.*)',                                         // 使用正则匹配选择器获取到的文本。match 都是填写正则表达式。
+//         description_element: '#zoomcon',                             // 正文。
+//         author_element: undefined,                                   // 正文来源，一般是某某网站、某某媒体，默认使用 <meta name="ContentSource" content="*">。
+//         author_match: undefined,                                     // 匹配正文来源。
+//         authorisme: '茂名市电白区人民政府网',                        // 作者是 本网 等内容时改成这里的文本。
+//         pubDate_element: 'publishtime',                              // 发布时间，默认使用 <meta name="PubDate" content="*">。
+//         pubDate_match: '(.*)',                                       // 匹配发布时间。
+//         pubDate_format: undefined                                    // 发布时间的格式。
+//     };
+//     await gdgov(info, ctx);
+// };
+
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const gdgov = async (info, ctx) => {
+    const path = ctx.path.split('/').filter((item) => item !== '');
+    const [site, branch] = path;
+
+    // 网站
+    const pathstartat = info.pathstartat === undefined ? 0 : info.pathstartat;
+    let rootUrl = 'http://' + branch + '.' + site + '.gov.cn';
+    for (let index = 0; index < pathstartat; index++) {
+        const element = path[index + 2];
+        rootUrl = rootUrl + '/' + element;
+    }
+
+    // 默认路径
+    const defaultPath = info.defaultPath;
+
+    // 网站名
+    let name_element = info.name_element;
+    const name_match_type = info.name_match_type;
+    const name_match = info.name_match;
+    const name_join = info.name_join;
+
+    // 列表元素
+    let list_element = info.list_element;
+    let list_include = info.list_include;
+    if (list_element === undefined) {
+        list_element = 'a[href*="content"]';
+        list_include = 'site';
+    }
+    if (list_include === 'site') {
+        list_element = list_element.split(',').filter((item) => item !== '');
+        for (let index = 0; index < list_element.length; index++) {
+            list_element[index] += '[href*="' + rootUrl.slice(7) + '"]';
+        }
+        list_element = list_element.join(',');
+    }
+
+    // 标题元素
+    let title_element = info.title_element;
+    let title_match = info.title_match;
+
+    // 作者（来源）元素
+    const author_element = info.author_element;
+    const author_match = info.author_match;
+    const authorisme = info.authorisme;
+
+    // 正文
+    let description_element = info.description_element;
+
+    // 发布时间元素
+    let pubDate_element = info.pubDate_element;
+    let pubDate_match = info.pubDate_match;
+    let pubDate_format = info.pubDate_format;
+
+    path.splice(0, 2 + pathstartat);
+    let pathname = path.join('/');
+    pathname = pathname === '' ? defaultPath : pathname.endsWith('/') ? pathname : pathname + '/';
+    const currentUrl = `${rootUrl}/${pathname}`;
+
+    let $ = '';
+    let name = '';
+    let list = '';
+    // 判断是否处于特殊目录
+    if (pathname.startsWith('gkmlpt')) {
+        title_element = undefined;
+        title_match = undefined;
+        description_element = 'div[class="article-content"]';
+        pubDate_element = undefined;
+        pubDate_match = undefined;
+        pubDate_format = undefined;
+
+        const res = await got(`${rootUrl}/gkmlpt/api/all/0`);
+        name = authorisme + '政府信息公开平台';
+        list = res.data.articles.filter((item) => item.url.includes('content'));
+    } else {
+        const res = await got(currentUrl);
+        const dataArray = res.data;
+        $ = cheerio.load(dataArray);
+        switch (name_match_type) {
+            case 'name':
+                name = name_element;
+                break;
+            case 'meta':
+                name_element = name_element.split(',').filter((item) => item !== '');
+                for (let index = 0; index < name_element.length; index++) {
+                    name_element[index] = $('meta[name="' + name_element[index].trim() + '"]').attr('content');
+                }
+                name = name_element.join(name_join);
+                break;
+            case 'element':
+                name_element = name_element.split(',').filter((item) => item !== '');
+                for (let index = 0; index < name_element.length; index++) {
+                    name_element[index] = $(name_element[index].trim()).text().match(name_match)[1];
+                }
+                name = name_element.join(name_join);
+                break;
+            default:
+                name = $('head title').text();
+                break;
+        }
+        list = $(list_element);
+    }
+
+    const lists = Array.from(
+        list.map((i, item) => {
+            let link = '';
+
+            if (pathname.startsWith('gkmlpt')) {
+                link = i.url;
+            } else {
+                link = $(item).attr('href');
+                // 判断获取到的链接是否完整，不完整则补全。
+                if (!link.startsWith('http')) {
+                    link.startsWith('/') ? (link = `${rootUrl}${link}`) : (link = `${rootUrl}/${link}`);
+                }
+            }
+
+            return link;
+        })
+    );
+
+    const items = await Promise.all(
+        lists.map((link) => {
+            const idlink = new URL(link);
+
+            if (idlink.pathname === '/zcjdpt') {
+                // http://www.dg.gov.cn/zcjdpt?id=4798
+                // http://smzj.maoming.gov.cn/zcjdpt?id=4595
+                // http://fgj.maoming.gov.cn/zcjdpt?id=4993
+                // https://zcjd.cloud.gd.gov.cn/api/home/article?id=4993
+                return ctx.cache.tryGet(link, async () => {
+                    const { art } = require('@/utils/render');
+
+                    const zcjdlink = 'https://zcjd.cloud.gd.gov.cn/api/home/article' + idlink.search;
+                    const response = await got(zcjdlink);
+                    const data = response.data.data;
+                    for (let index = 0; index < data.jie_du_items.length; index++) {
+                        data.jie_du_items[index].jd_content = data.jie_du_items[index].jd_content.replace(/((\n {4})|(\n))/g, '</p><p style="font-size: 16px;line-height: 32px;text-indent: 2em;">');
+                    }
+
+                    return {
+                        link,
+                        title: data.art_title,
+                        description: art(__dirname + '/templates/zcjdpt.art', data),
+                        pubDate: timezone(parseDate(data.pub_time), +8),
+                        author: RegExp('(本|本网|本站)').test(data.pub_unite) ? authorisme : data.pub_unite,
+                    };
+                });
+            } else if (idlink.host === 'mp.weixin.qq.com') {
+                const { finishArticleItem } = require('@/utils/wechat-mp');
+                return finishArticleItem(ctx, { link });
+            } else {
+                return ctx.cache.tryGet(link, async () => {
+                    // 获取网页
+                    const { data: res } = await got(link);
+                    const content = cheerio.load(res);
+
+                    // 获取来源
+                    let author = '';
+                    if (author_element === undefined) {
+                        author = content('meta[name="ContentSource"]').attr('content');
+                    } else {
+                        author = content(author_element).text().trim().match(author_match)[1].trim().replace(/(-*$)/g, '');
+                    }
+
+                    // 获取发布时间
+                    let pubDate = '';
+                    if (pubDate_element === undefined) {
+                        pubDate = content('meta[name="PubDate"]').attr('content');
+                    } else {
+                        pubDate = content(pubDate_element).text().trim().match(pubDate_match)[1].trim().replace(/(-*$)/g, '');
+                    }
+
+                    // 获取标题
+                    let title = '';
+                    if (title_element === undefined) {
+                        title = content('meta[name="ArticleTitle"]').attr('content');
+                    } else {
+                        title = content(title_element).text().trim().match(title_match)[1];
+                    }
+                    // 获取正文
+                    const description_content = description_element.split(',').filter((item) => item !== '');
+                    for (let index = 0; index < description_content.length; index++) {
+                        description_content[index] = content(description_content[index].trim()).html();
+                    }
+                    const description = description_content.join('');
+
+                    return {
+                        link,
+                        title,
+                        description,
+                        pubDate: timezone(parseDate(pubDate, pubDate_format), +8),
+                        author: RegExp('(本|本网|本站)').test(author) ? authorisme : author,
+                    };
+                });
+            }
+        })
+    );
+
+    ctx.state.data = {
+        title: name,
+        link: currentUrl,
+        item: items,
+    };
+};
+
+module.exports = {
+    gdgov,
+};