rsshub 1.0.0-master.f78ad03 → 1.0.0-master.f792555

package/lib/v2/gelonghui/subject.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const { parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const apiUrl = `https://www.gelonghui.com/api/subjects/${id}/contents`;
+    const pageUrl = `https://www.gelonghui.com/subject/${id}`;
+    const { data: response } = await got(pageUrl);
+    const { data } = await got(apiUrl, {
+        searchParams: {
+            isChoice: false,
+        },
+    });
+
+    const $ = cheerio.load(response);
+
+    const list = data.result.map((item) => ({
+        title: item.title,
+        description: item.summary,
+        link: item.link,
+        author: item.source,
+        pubDate: parseDate(item.timestamp, 'X'),
+    }));
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `格隆汇 - 主题 ${$('span.user-nick').text()} 的文章`,
+        description: $('div.user-name').parent().children('p').text(),
+        image: $('.subject-list-title').find('img').attr('data-src'),
+        link: `https://www.gelonghui.com/subject/${id}`,
+        item: items,
+    };
+};

package/lib/v2/gelonghui/user.js

@@ -0,0 +1,33 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const { parseItem } = require('./utils');
+
+module.exports = async (ctx) => {
+    const { id: userId } = ctx.params;
+    const currentUrl = 'https://www.gelonghui.com/api/community/dynamic/my-dynamics/v2';
+    const { data } = await got(currentUrl, {
+        searchParams: {
+            article: true,
+            userId,
+            count: ctx.query.limit ? Number(ctx.query.limit) : 15,
+        },
+    });
+
+    const list = data.result.map((item) => ({
+        title: item.title,
+        link: item.route,
+        description: item.content,
+        author: item.user.nick,
+        pubDate: parseDate(item.createTimestamp, 'X'),
+    }));
+
+    const items = await Promise.all(list.map((item) => parseItem(item, ctx.cache.tryGet)));
+
+    ctx.state.data = {
+        title: `格隆汇 - 用户 ${data.result[0].user.nick} 的文章`,
+        description: data.result.find((i) => i.user).user.brief,
+        image: data.result.find((i) => i.user).user.avatar.split('@')[0],
+        link: data.result.find((i) => i.user).user.route.replace('https://m.gelonghui.com', 'https://www.gelonghui.com'),
+        item: items,
+    };
+};

package/lib/v2/gelonghui/utils.js

@@ -0,0 +1,21 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+const parseItem = (item, tryGet) =>
+    tryGet(item.link, async () => {
+        const { data: res } = await got(item.link);
+        const $ = cheerio.load(res);
+        if (item.link.startsWith('https://www.gelonghui.com/live/')) {
+            item.title = $('.type-name').next().text().trim();
+            item.description = $('.dtb-content').html();
+        } else {
+            // article
+            item.title = $('.article-title').text().trim();
+            item.description = $('.article-summary').html() + $('article.article-with-html').html();
+        }
+        return item;
+    });
+
+module.exports = {
+    parseItem,
+};

package/lib/v2/gihyo/group.js

@@ -0,0 +1,48 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const groupId = ctx.params.id;
+
+    const baseUrl = 'https://gihyo.jp';
+    const url = `${baseUrl}/list/group/${groupId}`;
+
+    const response = await got(url);
+
+    const $ = cheerio.load(response.data);
+    const articles = $('ul[class=m-listitem]').find('li').not('.m-listitem--ad');
+    const title = $('head title').text();
+    const link = url;
+    const description = $('head meta[name=description]').attr('content');
+    const language = 'ja';
+
+    const item = articles
+        .map((_, article) => {
+            const _subtitle = $('p.m-listitem__title span.subtitle', article).text();
+            const _title = $('p.m-listitem__title', article)
+                .contents()
+                .filter((_, el) => el.nodeType === 3)
+                .text();
+            const title = `${_subtitle} ${_title}`;
+            const author = $('p.m-listitem__author', article).text();
+            const pubDate = timezone(parseDate($('span.date', article).text(), 'YYYY-MM-DD'), +9);
+            const link = `${baseUrl}${$('a', article).attr('href')}`.replace(/\?summary$/, '');
+            return {
+                title,
+                author,
+                pubDate,
+                link,
+            };
+        })
+        .get();
+
+    ctx.state.data = {
+        title,
+        link,
+        description,
+        language,
+        item,
+    };
+};

package/lib/v2/gihyo/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/list/group/:id': ['masakichi'],
+};

package/lib/v2/gihyo/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'gihyo.jp': {
+        _name: 'gihyo.jp',
+        '.': [
+            {
+                title: '記事一覧',
+                docs: 'https://docs.rsshub.app/programming.html#gihyo-jp',
+                source: '/list/group/:id',
+                target: '/gihyo/list/group/:id',
+            },
+        ],
+    },
+};

package/lib/v2/gihyo/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/list/group/:id', require('./group'));
+};

package/lib/v2/github/pulls.js

@@ -4,33 +4,32 @@ const md = require('markdown-it')({
     html: true,
     linkify: true,
 });
-const queryString = require('query-string');
 const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const user = ctx.params.user;
     const repo = ctx.params.repo;
     const state = ctx.params.state ?? 'open';
+    const labels = ctx.params.labels;
 
     const host = `https://github.com/${user}/${repo}/pulls`;
-    const url = `https://api.github.com/repos/${user}/${repo}/pulls`;
+    const url = `https://api.github.com/repos/${user}/${repo}/issues`; // every PR is also an issue
 
     const headers = { Accept: 'application/vnd.github.v3+json' };
     if (config.github && config.github.access_token) {
         headers.Authorization = `token ${config.github.access_token}`;
     }
-    const response = await got({
-        method: 'get',
-        url,
-        searchParams: queryString.stringify({
+    const response = await got(url, {
+        searchParams: {
             state,
+            labels,
             sort: 'created',
             direction: 'desc',
             per_page: ctx.query.limit ? (parseInt(ctx.query.limit) <= 100 ? parseInt(ctx.query.limit) : 100) : 100,
-        }),
+        },
         headers,
     });
-    const data = response.data;
+    const data = response.data.filter((item) => item.pull_request);
 
     ctx.state.data = {
         allowEmpty: true,

package/lib/v2/github/router.js

@@ -5,7 +5,7 @@ module.exports = function (router) {
     router.get('/contributors/:user/:repo/:order?/:anon?', require('./contributors'));
     router.get('/file/:user/:repo/:branch/:filepath+', require('./file'));
     router.get('/issue/:user/:repo/:state?/:labels?', require('./issue'));
-    router.get('/pull/:user/:repo/:state?', require('./pulls'));
+    router.get('/pull/:user/:repo/:state?/:labels?', require('./pulls'));
     router.get('/repos/:user', require('./repos'));
     router.get('/search/:query/:sort?/:order?', require('./search'));
     router.get('/starred_repos/:user', require('./starred_repos'));

package/lib/v2/github/topic.js

@@ -13,7 +13,7 @@ module.exports = async (ctx) => {
             .map((_, item) => {
                 item = $(item);
 
-                const title = $(item.find('h1 a').get(1)).attr('href').slice(1);
+                const title = $(item.find('h3 a').get(1)).attr('href').slice(1);
                 const author = title.split('/')[0];
                 const description = item.find('div.border-bottom > div > p + div').text();
                 const link = `https://github.com/${title}`;

package/lib/v2/github/trending.js

@@ -1,59 +1,91 @@
+const config = require('@/config').value;
 const got = require('@/utils/got');
 const { art } = require('@/utils/render');
 const cheerio = require('cheerio');
 const path = require('path');
 
 module.exports = async (ctx) => {
+    if (!config.github || !config.github.access_token) {
+        throw 'GitHub trending RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install/#pei-zhi-bu-fen-rss-mo-kuai-pei-zhi">relevant config</a>';
+    }
     const since = ctx.params.since;
     const language = ctx.params.language === 'any' ? '' : ctx.params.language;
     const spoken_language = ctx.params.spoken_language ?? '';
-    const url = `https://github.com/trending/${encodeURIComponent(language)}?since=${since}&spoken_language_code=${spoken_language}`;
 
-    const response = await got({
+    const trendingUrl = `https://github.com/trending/${encodeURIComponent(language)}?since=${since}&spoken_language_code=${spoken_language}`;
+    const { data: trendingPage } = await got({
         method: 'get',
-        url,
+        url: trendingUrl,
         headers: {
-            Referer: url,
+            Referer: trendingUrl,
         },
     });
+    const $ = cheerio.load(trendingPage);
 
-    const data = response.data;
-
-    const $ = cheerio.load(data);
-    const list = $('article');
-
-    const items = await Promise.all(
-        list.map((_, item) => {
-            item = $(item);
-            const endpoint = item.find('h1 a').attr('href');
-            const link = `https://github.com${endpoint}`;
-            return ctx.cache.tryGet(`github:trending:${endpoint}`, async () => {
-                const response = await got(link);
+    const articles = $('article');
+    const trendingRepos = articles
+        .map((_, item) => {
+            const [owner, name] = $(item).find('h1').text().split('/');
+            return {
+                name: name.trim(),
+                owner: owner.trim(),
+            };
+        })
+        .get();
 
-                const $ = cheerio.load(response.data);
-                const cover = $('meta[property="og:image"]');
+    const { data: repoData } = await got({
+        method: 'post',
+        url: 'https://api.github.com/graphql',
+        headers: {
+            Authorization: `bearer ${config.github.access_token}`,
+        },
+        json: {
+            query: `
+            query {
+            ${trendingRepos
+                .map(
+                    (repo, index) => `
+                _${index}: repository(owner: "${repo.owner}", name: "${repo.name}") {
+                    ...RepositoryFragment
+                }
+            `
+                )
+                .join('\n')}
+            }
 
-                const single = {
-                    title: item.find('h1').text(),
-                    author: item.find('h1').text().split('/')[0].trim(),
-                    description: art(path.join(__dirname, 'templates/trending-description.art'), {
-                        cover: cover.attr('content'),
-                        desc: item.find('p').text(),
-                        lang: item.find('span[itemprop="programmingLanguage"]').text() || 'Unknown',
-                        stars: item.find('.Link--muted').eq(0).text().trim(),
-                        forks: item.find('.Link--muted').eq(1).text().trim(),
-                    }),
-                    link,
-                };
+            fragment RepositoryFragment on Repository {
+                description
+                forkCount
+                nameWithOwner
+                openGraphImageUrl
+                primaryLanguage {
+                    name
+                }
+                stargazerCount
+            }
+            `,
+        },
+    });
 
-                return single;
-            });
-        })
-    );
+    const repos = Object.values(repoData.data).map((repo) => {
+        const found = trendingRepos.find((r) => `${r.owner}/${r.name}` === repo.nameWithOwner);
+        return { ...found, ...repo };
+    });
 
     ctx.state.data = {
         title: $('title').text(),
-        link: url,
-        item: items,
+        link: trendingUrl,
+        item: repos.map((r) => ({
+            title: r.nameWithOwner,
+            author: r.owner,
+            description: art(path.join(__dirname, 'templates/trending-description.art'), {
+                cover: r.openGraphImageUrl,
+                desc: r.description,
+                forks: r.forkCount,
+                lang: r.primaryLanguage?.name || 'Unknown',
+                stars: r.stargazerCount,
+            }),
+            link: `https://github.com/${r.nameWithOwner}`,
+        })),
     };
 };

package/lib/v2/good/index.js

@@ -0,0 +1,53 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const rootUrl = 'https://good.news';
+
+    const response = await got(rootUrl);
+
+    const $ = cheerio.load(response.data);
+
+    let items = $('.content-structure')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 50)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item
+                    .find('h4')
+                    .text()
+                    .replace(/^\s+\d{1,2}\./, ''),
+                link: item.parent().attr('href'),
+                pubDate: parseDate(item.find('.content-left').text()),
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got(item.link);
+
+                const content = cheerio.load(detailResponse.data);
+
+                content('img').each((_, img) => {
+                    if (img.attribs['data-src']) {
+                        img.attribs.src = img.attribs['data-src'].replace('-cdsb.onlycompress', '');
+                        delete img.attribs['data-src'];
+                    }
+                });
+
+                item.description = content('[contentscore]').html();
+
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'good.news - 今日要闻',
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/good/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['nczitzk'],
+};

package/lib/v2/good/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'good.news': {
+        _name: 'Good.news',
+        '.': [
+            {
+                title: '今日要闻',
+                docs: 'https://docs.rsshub.app/new-media.html#good-news-jin-ri-yao-wen',
+                source: ['/'],
+                target: '/good',
+            },
+        ],
+    },
+};

package/lib/v2/good/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index'));
+};

package/lib/v2/gov/beijing/bphc/index.js

@@ -0,0 +1,54 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+const mapping = {
+    '/project': {
+        title: '项目介绍',
+        list: '/front/index/project/page',
+        detail: '/front/index/project',
+        link: '/#/communityDetail?id=',
+    },
+    '/announcement': {
+        title: '通知公告',
+        list: '/front/announcement/page',
+        detail: '/front/announcement/',
+        link: '/#/announcementList/detail?id=',
+    },
+};
+
+module.exports = async (ctx) => {
+    const rootUrl = 'http://gycp.bphc.com.cn:21000';
+    const defaultPath = 'announcement';
+
+    const pathname = ctx.path.replace(/(^\/beijing\/bphc|\/$)/g, '');
+    const key = pathname === '' ? defaultPath : pathname.endsWith('/') ? pathname.slice(0, -1) : pathname;
+    const obj = mapping[key];
+    const currentUrl = `${rootUrl}${obj.list}`;
+
+    const listResp = await got(currentUrl).json();
+    const list = listResp.data?.records ?? [];
+    const items = await Promise.all(
+        list.map((item) => {
+            const detail = `${rootUrl}${obj.detail}/${item.id}`;
+            return ctx.cache.tryGet(detail, async () => {
+                const detailResponse = await got(detail).json();
+                const description = (detailResponse.data?.content || detailResponse.data?.introduction) ?? '';
+                const single = {
+                    title: item.title || item.fullName,
+                    author: description.match(/来源：(.*?)</)?.[1].trim() ?? item.operator,
+                    link: `${rootUrl}${obj.link}${item.id}`,
+                    description,
+                    pubDate: timezone(parseDate(item.createTime), +8),
+                };
+                return single;
+            });
+        })
+    );
+
+    ctx.state.data = {
+        title: obj.title,
+        link: rootUrl,
+        item: items,
+    };
+};

package/lib/v2/gov/ccdi/index.js

@@ -0,0 +1,18 @@
+const { rootUrl, parseNewsList, parseArticle } = require('./utils');
+
+module.exports = async (ctx) => {
+    const defaultPath = '/yaowenn/';
+
+    let pathname = ctx.path.replace(/(^\/ccdi|\/$)/g, '');
+    pathname = pathname === '' ? defaultPath : pathname.endsWith('/') ? pathname : pathname + '/';
+    const currentUrl = `${rootUrl}${pathname}`;
+
+    const { list, title } = await parseNewsList(currentUrl, '.list_news_dl li', ctx);
+    const items = await Promise.all(list.map((item) => parseArticle(item, ctx)));
+
+    ctx.state.data = {
+        title,
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/gov/ccdi/utils.js

@@ -0,0 +1,94 @@
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+
+const { CookieJar, Cookie } = require('tough-cookie');
+const cookieJar = new CookieJar();
+
+const owner = '中央纪委国家监委网站';
+const rootUrl = 'https://www.ccdi.gov.cn';
+const regex = /(?<key>[A-Z_]+)=(?<value>(?:.*?(?=; max-age)|[a-fA-F0-9]+))/gm;
+
+const parseCookie = async (body) => {
+    let m;
+    const cookies = [];
+    while ((m = regex.exec(body)) !== null) {
+        // This is necessary to avoid infinite loops with zero-width matches
+        if (m.index === regex.lastIndex) {
+            regex.lastIndex++;
+        }
+        const { key, value } = m.groups;
+        cookies.push(new Cookie({ key, value }));
+    }
+    await Promise.all(cookies.map((c) => cookieJar.setCookie(c, rootUrl)));
+};
+
+const parseNewsList = async (url, selector, ctx) => {
+    const response = await got(url, { cookieJar });
+    const data = response.data;
+    await parseCookie(data);
+
+    const $ = cheerio.load(data);
+    const list = $(selector)
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 20)
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').first().text().trim(),
+                link: new URL(item.find('a').first().attr('href'), url).href,
+                pubDate: parseDate(item.find('span').text(), 'YYYY-MM-DD'),
+            };
+        });
+    const title = $('.other_Location')
+        .text()
+        .replace(/(.+)首页/, owner);
+    return { list, title };
+};
+
+const changeTrCookie = async () => {
+    const cookies = await cookieJar.getCookies(rootUrl);
+    const c = cookies.find((c) => c.key === 'HOY_TR');
+    if (c) {
+        const value = c.value;
+        const tr_array = value.split(',');
+        const csr = tr_array[0];
+        const cnv = tr_array[1].split('');
+        const otr = tr_array[2].split('');
+        otr[0] = csr.charAt(parseInt(cnv[0], 16));
+        const nc = new Cookie({ key: 'HOY_TR', value: csr + ',' + cnv.join('') + ',' + otr.join('') + ',0' });
+        await cookieJar.setCookie(nc, rootUrl);
+    }
+};
+
+const parseArticle = async (item, ctx) => {
+    await changeTrCookie();
+    return await ctx.cache.tryGet(item.link, async () => {
+        const response = await got(item.link, { cookieJar });
+        const data = response.data;
+        await parseCookie(data);
+
+        const $ = cheerio.load(data);
+        const title = $('.daty, .source-box').text().trim();
+        item.author = title.match(/来源：(.*)发布时间/s)?.[1].trim() ?? owner;
+        item.pubDate = timezone(parseDate(title.match(/发布时间：(.*)分享/s)?.[1].trim() ?? item.pubDate), +8);
+
+        // Change the img src from relative to absolute for a better compatibility
+        $('.content, .bom-box')
+            .find('img')
+            .each((_, el) => {
+                $(el).attr('src', new URL($(el).attr('src'), item.link).href);
+                // oldsrc is causing freshrss imageproxy not to work correctly
+                $(el).removeAttr('oldsrc').removeAttr('alt');
+            });
+        item.description = $('.content, .bom-box').html();
+        return item;
+    });
+};
+
+module.exports = {
+    rootUrl,
+    parseNewsList,
+    parseArticle,
+};

package/lib/{routes/sse/sserules.js → v2/gov/cnnic/index.js}

@@ -1,11 +1,14 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const slug = ctx.params.slug || 'latest';
+    ctx.path = ctx.path.replace(/^\/cnnic/g, '');
+
+    const rootUrl = 'http://www.cnnic.net.cn';
+    const currentUrl = `${rootUrl}${ctx.path === '/' ? '/gywm/xwzx/rdxw/20172017_7086/' : `${ctx.path}/`}`;
 
-    const rootUrl = 'http://www.sse.com.cn';
-    const currentUrl = `${rootUrl}/lawandrules/sserules/${slug.replace(/-/g, '/')}`;
     const response = await got({
         method: 'get',
         url: currentUrl,
@@ -13,28 +16,30 @@ module.exports = async (ctx) => {
 
     const $ = cheerio.load(response.data);
 
-    const list = $('.sse_list_1 dl dd a')
-        .slice(0, 15)
-        .map((_, item) => {
+    let items = $('.link a')
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 12)
+        .toArray()
+        .map((item) => {
             item = $(item);
+
             return {
                 title: item.text(),
-                link: `${rootUrl}${item.attr('href')}`,
-                pubDate: new Date(item.prev().text()).toUTCString(),
+                link: new URL(item.attr('href'), currentUrl).href,
             };
-        })
-        .get();
+        });
 
-    const items = await Promise.all(
-        list.map((item) =>
+    items = await Promise.all(
+        items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 const detailResponse = await got({
                     method: 'get',
                     url: item.link,
                 });
+
                 const content = cheerio.load(detailResponse.data);
 
-                item.description = content('.allZoom').html();
+                item.description = content('.TRS_Editor').html();
+                item.pubDate = timezone(parseDate(content('.info .text span').first().text(), 'YYYY年MM月DD日 HH:mm'), +8);
 
                 return item;
             })