rsshub 1.0.0-master.fd8cb4a → 1.0.0-master.fdba2d7

package/lib/routes/coolapk/hot.js

@@ -0,0 +1,99 @@
+const got = require('@/utils/got');
+const utils = require('./utils');
+const getLinkAndTitle = (type, period) => {
+    const baseURL = 'https://api.coolapk.com/v6/page/dataList?url=';
+    let link;
+    const res = {};
+    const types = {
+        jrrm: {
+            title: '今日热门',
+            url: baseURL + '%2Ffeed%2FstatList%3FcacheExpires%3D300%26statType%3Dday%26sortField%3Ddetailnum%26title%3D%E4%BB%8A%E6%97%A5%E7%83%AD%E9%97%A8&title=%E4%BB%8A%E6%97%A5%E7%83%AD%E9%97%A8&subTitle=&page=1',
+        },
+
+        dzb: {
+            title: '点赞榜',
+            sortField: 'likenum',
+        },
+
+        scb: {
+            title: '收藏榜',
+            sortField: 'favnum',
+        },
+        plb: {
+            title: '评论榜',
+            sortField: 'replynum',
+        },
+        ktb: {
+            title: '酷图榜',
+            sortField: 'likenum',
+        },
+    };
+
+    const periods = {
+        daily: {
+            description: '日榜',
+            statType: 'day',
+        },
+        weekly: {
+            description: '周榜',
+            statType: '7days',
+        },
+    };
+
+    if (type === 'jrrm') {
+        res.link = types.jrrm.url;
+        res.title = types.jrrm.title;
+        return res;
+    } else if (type === 'ktb') {
+        const trans = {
+            daily: {
+                description: '周榜',
+                statDays: '7days',
+            },
+            weekly: {
+                description: '月榜',
+                statDays: '30days',
+            },
+        };
+        link = `#/feed/coolPictureList?statDays=` + trans[period].statDays + `&listType=statFavNum&buildCard=1&title=` + trans[period].description + `&page=1`;
+        res.title = '酷图榜-' + trans[period].description;
+    } else {
+        link = `#/feed/statList?statType=` + periods[period].statType + `&sortField=` + types[type].sortField + `&title=` + periods[period].description + `&page=1`;
+        res.title = types[type].title + `-` + periods[period].description;
+    }
+    res.link = baseURL + encodeURIComponent(link);
+    return res;
+};
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type || 'jrrm';
+    const period = ctx.params.period || 'daily';
+    const { link, title } = getLinkAndTitle(type, period);
+    const r = await got({
+        method: 'get',
+        url: link,
+        headers: utils.getHeaders(),
+    });
+    const data = r.data.data;
+    const t = [];
+    for (const i of data) {
+        if (i.entityType === 'card') {
+            for (const k of i.entities) {
+                t.push(k);
+            }
+        } else {
+            t.push(i);
+        }
+    }
+
+    let out = await Promise.all(t.map((item) => utils.parseDynamic(item, ctx)));
+
+    out = out.filter((i) => i);
+
+    ctx.state.data = {
+        title,
+        link: 'https://www.coolapk.com/',
+        description: `热榜-` + title,
+        item: out,
+    };
+};

package/lib/routes/coolapk/huati.js

@@ -11,18 +11,7 @@ module.exports = async (ctx) => {
     });
     const data = response.data.data;
 
-    let out = await Promise.all(
-        data.map(async (item) => {
-            if (item.type === 12) {
-                // 图文内容
-                return utils.parseTuwen(item, ctx);
-            } else if (String(item.type) in utils.dynamicTpye) {
-                return utils.parseDongtai(item, ctx);
-            } else {
-                return null;
-            }
-        })
-    );
+    let out = await Promise.all(data.map((item) => utils.parseDynamic(item, ctx)));
 
     out = out.filter((i) => i); // 去除空值
     if (out.length === 0) {

package/lib/routes/coolapk/toutiao.js

@@ -0,0 +1,31 @@
+const got = require('@/utils/got');
+const utils = require('./utils');
+
+module.exports = async (ctx) => {
+    const type = ctx.params.type || 'history';
+    const urls = {
+        history: {
+            url: 'https://api.coolapk.com/v6/page/dataList?url=%23%2ffeed%2fheadlineV8List%3ftype%3d0%2c5%2c9%2c8%2c12%2c10%2c11%2c13%26title%3d%e5%8e%86%e5%8f%b2%e5%a4%b4%e6%9d%a1%26page%3d1',
+            title: '历史头条',
+        },
+        latest: {
+            url: 'https://api.coolapk.com/v6/page/dataList?url=%23%2ffeed%2fdigestList%3ftype%3d0%2c5%2c12%2c10%2c11%2c13%2c8%2c9%26title%3d%e6%9c%80%e6%96%b0%e5%8a%a8%e6%80%81%26page%3d1',
+            title: '最新动态',
+        },
+    };
+    const response = await got({
+        method: 'get',
+        url: urls[type].url,
+        headers: utils.getHeaders(),
+    });
+    const data = response.data.data;
+
+    const out = await Promise.all(data.map((item) => utils.parseDynamic(item, ctx)));
+
+    ctx.state.data = {
+        title: urls[type].title,
+        link: 'https://www.coolapk.com/',
+        description: urls[type].title,
+        item: out,
+    };
+};

package/lib/routes/coolapk/tuwen.js

@@ -2,8 +2,9 @@ const got = require('@/utils/got');
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
+    const type = ctx.params.type || 'hot';
     let full_url, feedTitle;
-    if (ctx.originalUrl.startsWith(`/coolapk/tuwen-xinxian`)) {
+    if (ctx.originalUrl.startsWith(`/coolapk/tuwen-xinxian`) || type === 'latest') {
         // 实时
         full_url =
             utils.base_url +
@@ -21,12 +22,12 @@ module.exports = async (ctx) => {
     });
     const data = response.data.data;
 
-    const out = await Promise.all(data.map(async (item) => utils.parseTuwen(item, ctx)));
+    const out = await Promise.all(data.map((item) => utils.parseDynamic(item, ctx)));
 
     ctx.state.data = {
-        title: `${feedTitle}`,
+        title: feedTitle,
         link: 'https://www.coolapk.com/',
-        description: `${feedTitle}`,
+        description: feedTitle,
         item: out,
     };
 };

package/lib/routes/coolapk/userDynamic.js

@@ -3,7 +3,7 @@ const utils = require('./utils');
 
 module.exports = async (ctx) => {
     const uid = ctx.params.uid;
-    const full_url = utils.base_url + `/v6/user/feedList?uid=${uid}&page=1&showAnonymous=0`;
+    const full_url = utils.base_url + `/v6/user/feedList?uid=${uid}&page=1&showAnonymous=0&isIncludeTop=1&showDoing=1`;
     let username;
     const response = await got({
         method: 'get',
@@ -15,19 +15,12 @@ module.exports = async (ctx) => {
         throw Error('这个人没有任何动态。');
     }
     let out = await Promise.all(
-        data.map(async (item) => {
+        data.map((item) => {
             if (!username) {
                 username = item.username;
             }
 
-            if (item.type === 12) {
-                // 图文内容
-                return utils.parseTuwen(item, ctx);
-            } else if (String(item.type) in utils.dynamicTpye) {
-                return utils.parseDongtai(item, ctx);
-            } else {
-                return null;
-            }
+            return utils.parseDynamic(item, ctx);
         })
     );
 

package/lib/routes/coolapk/utils.js

@@ -2,7 +2,7 @@ const md5 = require('@/utils/md5');
 const cheerio = require('cheerio');
 const got = require('@/utils/got');
 
-const dynamicTpye = { 0: '基本动态', 9: '评论', 20: '交易动态' };
+const dynamicTpye = { 0: '基本动态', 8: '酷图', 9: '评论', 10: '提问', 11: '回答', 12: '图文', 15: '二手', 17: '观点', 20: '交易动态' };
 
 const getRandomDEVICE_ID = () => {
     let id = [8, 4, 4, 4, 12];
@@ -29,15 +29,21 @@ const getHeaders = () => ({
     'X-App-Token': get_app_token(),
     'X-Sdk-Int': '29',
     'X-Sdk-Locale': 'zh-CN',
-    'X-App-Version': '10.4',
-    'X-App-Code': '2007081',
-    'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 10; Redmi K30 5G MIUI/V12.0.3.0.QGICMXM) (#Build; Redmi; Redmi K30 5G; QKQ1.191222.002 test-keys; 10) +CoolMarket/10.4-2007081',
+    'X-App-Version': '11.0',
+    'X-Api-Version': '11',
+    'X-App-Code': '2101202',
+    'User-Agent': 'Dalvik/2.1.0 (Linux; U; Android 10; Redmi K30 5G MIUI/V12.0.3.0.QGICMXM) (#Build; Redmi; Redmi K30 5G; QKQ1.191222.002 test-keys; 10) +CoolMarket/11.0-2101202',
 });
 
 const parseTuwenFromRaw = (raw) =>
     raw.map((i) => {
         if (i.type === 'text') {
-            return `<p>${i.message}</p>`;
+            const output = i.message
+                .split('\n')
+                .filter((t) => t !== '')
+                .map((t) => `<p>${t}</p>`)
+                .join('');
+            return output;
         } else if (i.type === 'image') {
             return `<div class="img-container" style="text-align: center;">
                 <img src="${i.url}">
@@ -48,64 +54,89 @@ const parseTuwenFromRaw = (raw) =>
         }
     });
 
-const parseDongtai = async (item, ctx) => {
-    const author = item.username;
-    const pubdate = new Date(item.lastupdate * 1000).toUTCString();
-    const itemUrl = item.shareUrl.split('?')[0];
-    let description;
-    if (item.issummary) {
-        // 需要爬内容
-        description = await ctx.cache.tryGet(itemUrl, async () => {
-            const result = await got({
-                method: 'get',
-                url: itemUrl,
-                headers: getHeaders(),
-            });
-            const message = `<p>` + result.data.data.message.split('\n').join('<br>') + `</p>`;
-            const picArr = item.picArr.filter((i) => i).map((i) => `<img src="${i}">`); // 若无图片，item.picArr=[""]
-            return message + picArr.join('');
-        });
-    } else {
-        const picArr = item.picArr.filter((i) => i).map((i) => `<img src="${i}">`);
-        description = `<p>` + item.message + `</p>` + picArr.join('');
+const parseDynamic = async (item, ctx) => {
+    const pubDate = new Date(item.lastupdate * 1000).toUTCString();
+    if (item.entityType === 'sponsorCard') {
+        return;
     }
-    const $ = cheerio.load('<div class="title-filter">' + description + '</div>');
-    const title = $('.title-filter').text().slice(0, 30).trim() + '...';
-    return {
-        title: title,
-        description: description,
-        pubDate: pubdate,
-        link: itemUrl,
-        author: author,
-    };
-};
+    const itemUrl = item.shareUrl.split('?')[0];
+    let description, title;
+    const type = parseInt(item.type);
+    switch (type) {
+        case 0:
+        case 8:
+        case 9:
+        case 10:
+        case 11:
+        case 15:
+        case 17:
+        case 20: {
+            // //////////////////////////////////////////// 基本内容 ////////////////////////////////////////////
+            if (item.issummary) {
+                // 需要爬内容
+                description = await ctx.cache.tryGet(itemUrl, async () => {
+                    const result = await got({
+                        method: 'get',
+                        url: itemUrl,
+                        headers: getHeaders(),
+                    });
+                    const message = `<p>` + result.data.data.message.split('\n').join('<br>') + `</p>`;
+                    const picArr = item.picArr.filter((i) => i).map((i) => `<img src="${i}">`); // 若无图片，item.picArr=[""]
+                    return message + picArr.join('');
+                });
+            } else {
+                const picArr = item.picArr.filter((i) => i).map((i) => `<img src="${i}">`);
+                description = `<p>` + item.message + `</p>` + picArr.join('');
+            }
+            const $ = cheerio.load('<div class="title-filter">' + description + '</div>');
+            title = $('.title-filter').text().slice(0, 30).trim() + '...';
 
-const parseTuwen = async (item, ctx) => {
-    const title = item.title;
-    const pubdate = new Date(item.lastupdate * 1000).toUTCString();
-    const author = item.username;
+            // //////////////////////////////////////////// 基本内容结束 ////////////////////////////////////////////
 
-    const description = await ctx.cache.tryGet(item.shareUrl, async () => {
-        const result = await got({
-            method: 'get',
-            url: item.shareUrl,
-            headers: getHeaders(),
-        });
+            if (type === 17) {
+                const keys = item.extra_key.split(',');
+                description += `<p>` + item.vote.message_title + ` 已选${keys.length}项</p>`;
+                for (const i of item.vote.options) {
+                    if (keys.includes(String(i.id))) {
+                        description += `<p>${i.title}√</p>`;
+                    }
+                }
+            } else if (type === 10 || type === 11) {
+                title = `${item.message_title} 更多:` + title;
+            }
 
-        const raw = JSON.parse(result.data.data.message_raw_output);
+            break;
+        }
+        case 12: {
+            title = item.title;
+            description = await ctx.cache.tryGet(itemUrl, async () => {
+                const result = await got({
+                    method: 'get',
+                    url: itemUrl,
+                    headers: getHeaders(),
+                });
 
-        const tags = parseTuwenFromRaw(raw);
+                const raw = JSON.parse(result.data.data.message_raw_output);
 
-        return `<img src="${result.data.data.pic}"><hr>` + tags.join('');
-    });
+                const tags = parseTuwenFromRaw(raw);
+
+                return `<img src="${result.data.data.pic}"><hr>` + tags.join('');
+            });
+            break;
+        }
+        default:
+            // console.log(item.type);
+            // console.log(item);
+            break;
+    }
 
     return {
-        title: title,
-        description: description,
-        pubDate: pubdate,
+        title,
+        description,
+        pubDate,
         link: item.shareUrl,
-        guid: item.shareUrl.split('?')[0],
-        author: author,
+        guid: itemUrl,
+        author: item.username,
     };
 };
 
@@ -113,7 +144,6 @@ module.exports = {
     get_app_token,
     base_url,
     getHeaders,
-    parseTuwen,
-    parseDongtai,
+    parseDynamic,
     dynamicTpye,
 };

package/lib/routes/copymanga/comic.js

@@ -0,0 +1,67 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+// 直接调用拷贝漫画的接口
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+
+    // 获取漫画列表
+    let bHasNextPage = false;
+    const iReqLimit = (ctx.queries && ctx.queries.limit) || 100;
+    let iReqOffSet = 0;
+    const strBaseUrl = `https://api.copymanga.com/api/v3/comic/${id}/group/default/chapters?`;
+    let chapterArray = [];
+
+    do {
+        bHasNextPage = false;
+        // eslint-disable-next-line no-await-in-loop
+        const { data } = await got.get(strBaseUrl + 'limit=' + iReqLimit + '&offset=' + iReqOffSet);
+        const { code, results } = data;
+
+        if (code !== 200) {
+            break;
+        }
+
+        if (results.limit + results.offset < results.total) {
+            bHasNextPage = true;
+        }
+        iReqOffSet += iReqLimit;
+
+        chapterArray = chapterArray.concat(results.list);
+    } while (bHasNextPage);
+
+    chapterArray = chapterArray
+        .map(({ comic_path_word, uuid, name, size, datetime_created, index }) => ({
+            link: 'https://copymanga.com/comic/' + comic_path_word + '/chapter/' + uuid,
+            title: name,
+            num: size,
+            updateTime: datetime_created,
+            index,
+        }))
+        .reverse();
+
+    // 获取漫画标题、介绍
+    const { bookTitle, bookIntro } = await ctx.cache.tryGet(`https://copymanga.com/comic/${id}`, async () => {
+        const { data } = await got.get(`https://copymanga.com/comic/${id}`);
+        const $ = cheerio.load(data);
+        return {
+            bookTitle: $('.comicParticulars-title-right > ul > li > h6').text(),
+            bookIntro: $('.intro').text(),
+        };
+    });
+
+    const genResult = (chapter) => ({
+        link: chapter.link,
+        title: chapter.title,
+        description: `
+            <h1>${chapter.num}p</h1>
+        `.trim(),
+    });
+
+    ctx.state.data = {
+        title: `拷贝漫画 - ${bookTitle}`,
+        link: `https://copymanga.com/comic/${id}`,
+        description: bookIntro,
+        item: chapterArray.map(genResult),
+    };
+};

package/lib/routes/coronavirus/sg-moh.js

@@ -19,7 +19,7 @@ module.exports = async (ctx) => {
         })
         .map((_, item) => {
             item = $(item);
-            const title = `${item.find('a').first().text()}`;
+            const title = item.find('a').first().text();
             const dateRaw = item.find('td:first-child').text().trim();
             const pubDate = Date.parse(dateRaw) && dateRaw;
             const link = item.find('a').attr('href');

package/lib/routes/coronavirus/yahoo-japan.js

@@ -0,0 +1,77 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const dayjs = require('dayjs');
+
+module.exports = async (ctx) => {
+    const tdfk = ctx.params.tdfk || false;
+    const uri = tdfk ? `https://news.yahoo.co.jp/pages/article/covid19${tdfk}` : `https://news.yahoo.co.jp/pages/article/20200207`;
+    const req_header = { 'User-Agent': 'Mozilla/5.0 (Linux; Android 9; SM-G960F Build/PPR1.180610.011; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.157 Mobile Safari/537.36' };
+
+    const resp = await got({
+        method: 'get',
+        url: uri,
+        headers: req_header,
+    });
+
+    const $ = cheerio.load(resp.data);
+
+    const this_year = dayjs().year();
+    const this_month = dayjs().month() + 1;
+
+    const art_uri = [];
+    $('#layoutFooter ul.dlpThumbLink a').each((i, e) => {
+        const link = $(e).attr('href');
+        const text = $(e).find('.dlpThumbText span').eq(0).text();
+        const author = $(e).find('.dlpQuote').text();
+        const date = $(e).find('.dlpDate').text() + ' +9'; // explicit timezone
+
+        let date_obj = dayjs(date).year(this_year);
+
+        if (date_obj.month() + 1 > this_month) {
+            // if the article is from the last year
+            date_obj = date_obj.year(this_year - 1);
+        }
+
+        art_uri.push({ l: link, t: text, a: author, d: date_obj.toString() });
+    });
+
+    const getNews = async (uri) => {
+        const ret = {
+            link: uri.l,
+            title: uri.t,
+            author: uri.a,
+            pubDate: uri.d,
+            description: null,
+        };
+
+        if (!uri.l.includes('//news.yahoo.co.jp')) {
+            return ret; // do not process uncertain pages
+        }
+
+        const page_data = await ctx.cache.tryGet(uri.l, async () => {
+            const resp = await got({
+                method: 'get',
+                url: uri.l,
+                headers: req_header,
+            });
+            return resp.data;
+        });
+
+        const $doc = cheerio.load(page_data);
+
+        const iso_date = $doc('meta[name="pubdate"]').attr('content');
+        ret.pubDate = dayjs(iso_date).toString();
+        ret.description = $doc('div.article_body').html() || $doc('meta[name="description"]').attr('content');
+
+        return ret;
+    };
+
+    const items = await Promise.all(art_uri.map(getNews));
+
+    ctx.state.data = {
+        title: $('title').text(),
+        link: uri,
+        description: $('meta[name="description"]').attr('content'),
+        item: items,
+    };
+};

package/lib/routes/cpta/notice.js

@@ -20,17 +20,16 @@ module.exports = async (ctx) => {
 
     ctx.state.data = {
         title: '中国人事考试网 - 通知公告',
-        link: link,
+        link,
         item: await Promise.all(
-            list.map(
-                async (item) =>
-                    await ctx.cache.tryGet(item.link, async () => {
-                        const detailResponse = await got({ method: 'get', url: item.link, responseType: 'buffer' });
-                        const content = cheerio.load(iconv.decode(detailResponse.data, 'gbk'));
-                        item.description = content('div.text').html();
-                        item.pubDate = new Date(content('#p_publishtime').text().replace(/[年|月]/g, '-').replace(/日/g, ' ') + ' GMT+8').toUTCString();
-                        return item;
-                    })
+            list.map((item) =>
+                ctx.cache.tryGet(item.link, async () => {
+                    const detailResponse = await got({ method: 'get', url: item.link, responseType: 'buffer' });
+                    const content = cheerio.load(iconv.decode(detailResponse.data, 'gbk'));
+                    item.description = content('div.text').html();
+                    item.pubDate = new Date(content('#p_publishtime').text().replace(/[年|月]/g, '-').replace(/日/g, ' ') + ' GMT+8').toUTCString();
+                    return item;
+                })
             )
         ),
     };

package/lib/routes/creaders/headline.js

@@ -24,17 +24,16 @@ module.exports = async (ctx) => {
         .get();
 
     const items = await Promise.all(
-        list.map(
-            async (item) =>
-                await ctx.cache.tryGet(item.link, async () => {
-                    const res = await got.get(item.link, {
-                        responseType: 'buffer',
-                    });
-                    const content = cheerio.load(iconv.decode(res.data, 'gbk'));
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const res = await got.get(item.link, {
+                    responseType: 'buffer',
+                });
+                const content = cheerio.load(iconv.decode(res.data, 'gbk'));
 
-                    item.description = content('#newsContent').html();
-                    return item;
-                })
+                item.description = content('#newsContent').html();
+                return item;
+            })
         )
     );
 

package/lib/routes/cria/news.js

@@ -0,0 +1,57 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    const id = ctx.params.id || '';
+
+    const rootUrl = 'http://www.cria.org.cn';
+    const currentUrl = `${rootUrl}/${id ? `newslist/${id}.html` : 'news'}`;
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('.ch_news,.enterprise_news_c')
+        .find('ul li')
+        .map((_, item) => {
+            item = $(item);
+            const a = item.find('a');
+
+            return {
+                link: `${rootUrl}${a.attr('href')}`,
+            };
+        })
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                try {
+                    const detailResponse = await got({
+                        method: 'get',
+                        url: item.link,
+                    });
+                    const content = cheerio.load(detailResponse.data);
+
+                    item.title = content('title').text();
+                    item.description = content('.news_details_c').html();
+                    item.pubDate = new Date(detailResponse.data.match(/<span>发布时间：(.*)<\/span><span>来源:/)[1]).toUTCString();
+
+                    return item;
+                } catch (e) {
+                    return Promise.resolve('');
+                }
+            })
+        )
+    );
+
+    const title = id ? response.data.match(/<h3><span style="margin: 10px; font-size:15px;">当前位置：中国橡胶网 > (.*)<\/span><\/h3>/)[1] : '新闻资讯';
+
+    ctx.state.data = {
+        title: `${title} - 中国橡胶网`,
+        link: currentUrl,
+        item: items,
+    };
+};