rsshub 1.0.0-master.f97cdb2 → 1.0.0-master.fa3428f

package/lib/{routes → v2}/cctv/utils/news.js

@@ -1,50 +1,50 @@
 const got = require('@/utils/got');
 const path = require('path');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const UA = require('@/utils/rand-user-agent')({ browser: 'mobile safari', os: 'ios', device: 'mobile' });
 
 module.exports = async (category, ctx) => {
-    const url = `http://news.cctv.com/2019/07/gaiban/cmsdatainterface/page/${category}_1.jsonp`;
+    const url = `https://news.cctv.com/2019/07/gaiban/cmsdatainterface/page/${category}_1.jsonp`;
 
     const response = await got({
         method: 'get',
         url,
         headers: {
             Referer: `http://news.cctv.com/${category}`,
+            'User-Agent': UA,
         },
     });
 
     const data = JSON.parse(response.data.slice(category.length + 1, -1));
     const list = data.data.list;
     const resultItem = await Promise.all(
-        list.map(async ({ title, url, focus_date, image }) => {
-            const item = {
-                title,
-                link: url,
-                pubDate: new Date(focus_date).toUTCString(),
-            };
-            const key = `cctv-news: ${url}`;
-            const value = await ctx.cache.get(key);
-
-            if (value) {
-                item.description = value;
-            } else {
+        list.map(({ title, url, focus_date, image }) =>
+            ctx.cache.tryGet(`cctv-news: ${url}`, async () => {
+                const item = {
+                    title,
+                    link: url,
+                    pubDate: timezone(parseDate(focus_date), +8),
+                };
                 const id = path.parse(url).name;
                 const unknownTip = '未知类型，请点击<a href="https://github.com/DIYgod/RSSHub/issues">链接</a>提交issue';
                 let description;
                 let api;
                 let type;
+                let author;
 
                 if (id.startsWith('ART')) {
                     // 普通新闻
-                    api = `http://api.cntv.cn/Article/getXinwenNextArticleInfo?serviceId=sjnews&id=${id}&t=json`;
+                    api = `https://api.cntv.cn/Article/getXinwenNextArticleInfo?serviceId=sjnews&id=${id}&t=json`;
                     type = 'ART';
                 } else if (id.startsWith('PHO')) {
                     // 图片
-                    api = `http://api.cntv.cn/Article/contentinfo?id=${id}&serviceId=sjnews&t=json`;
+                    api = `https://api.cntv.cn/Article/contentinfo?id=${id}&serviceId=sjnews&t=json`;
                     type = 'PHO';
                 } else if (id.startsWith('VIDE')) {
                     // 视频
                     const vid = path.parse(image).name.split('-')[0];
-                    api = `http://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid=${vid}`;
+                    api = `https://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid=${vid}`;
                     type = 'VIDE';
                 } else {
                     // 未识别
@@ -56,13 +56,14 @@ module.exports = async (category, ctx) => {
                         method: 'get',
                         url: api,
                         headers: {
-                            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
+                            'User-Agent': UA,
                         },
                     });
 
                     switch (type) {
                         case 'ART':
                             description = data.article_content;
+                            author = data.article_source;
                             break;
 
                         case 'PHO':
@@ -74,10 +75,12 @@ module.exports = async (category, ctx) => {
                                 `;
                                 return description;
                             }, '');
+                            author = data.source;
                             break;
 
                         case 'VIDE':
                             description = `<video src="${data.hls_url}" controls="controls" poster="${image}" style="width: 100%"></video>`;
+                            author = data.article_source;
                             break;
 
                         default:
@@ -86,16 +89,16 @@ module.exports = async (category, ctx) => {
                 }
 
                 item.description = description;
-                ctx.cache.set(key, item.description);
-            }
+                item.author = author;
 
-            return item;
-        })
+                return item;
+            })
+        )
     );
 
     return {
         title: `央视新闻 ${category}`,
-        link: `http://news.cctv.com/${category}`,
+        link: `https://news.cctv.com/${category}`,
         description: `央视新闻 ${category}`,
         item: resultItem,
     };

package/lib/{routes → v2}/cctv/utils/xinwen1j1.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 //
 // 测试http://localhost:1200/cctv/xinwen1j1
 
@@ -20,20 +21,19 @@ async function load(link) {
 
     // poster="?" 以后再弄
     // <video src="${videoDetail.hls_url}" controls="controls"  poster="?"  style="width: 100%"></video>
-    const description = `<video src="${videoDetail.hls_url}" controls="controls"  style="width: 100%"></video>`;
+    const description = `${$('meta[name=description]').attr('content')}<br><video src="${videoDetail.hls_url}" controls="controls"  style="width: 100%"></video>`;
 
     // 提取内容
     return { description };
 }
 
 // 同步 列表
-const ProcessFeed = (list, caches) =>
+const ProcessFeed = (data, ctx) =>
     Promise.all(
-        list.map(async (item) => {
-            const $ = cheerio.load(item);
-            const title = $('a').text();
-            const itemUrl = $('a').attr('href').split('?')[0];
-            const itemDate = new Date(itemUrl.split('/VI')[0].split('com/')[1]).toUTCString();
+        data.list.map(async (data) => {
+            const title = data.title;
+            const itemUrl = data.url;
+            const itemDate = parseDate(data.focus_date);
 
             // 列表上提取到的信息
             const single = {
@@ -46,27 +46,30 @@ const ProcessFeed = (list, caches) =>
 
             // 使用tryGet方法从缓存获取内容。
             // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。
-            const other = await caches.tryGet(itemUrl, () => load(itemUrl));
+            const other = await ctx.cache.tryGet(itemUrl, () => load(itemUrl));
 
             // 合并解析后的结果集作为该篇文章最终的输出结果
-            return Promise.resolve(Object.assign({}, single, other));
+            return { ...single, ...other };
         })
     );
-module.exports = async (category, ctx) => {
-    const baseUrl = 'http://tv.cctv.com/lm/xinwen1j1/videoset/index.shtml';
+module.exports = async (ctx) => {
+    const baseUrl = 'https://api.cntv.cn/NewVideo/getVideoListByColumn?id=TOPC1451559066181661&n=20&sort=desc&p=1&mode=0&serviceId=tvcctv';
     // 获取要处理的页面
-    const res = await got({ method: 'get', url: baseUrl });
+    const res = await got({
+        method: 'get',
+        url: baseUrl,
+        headers: {
+            Referer: 'https://tv.cctv.com/lm/xinwen1j1/videoset/index.shtml#&Type=0',
+        },
+    });
 
-    const $ = cheerio.load(res.data);
+    const data = res.data.data;
 
-    let list = $('div.text').slice(0, 30); // 取前30个
     // $('div.image').slice(0,3).find('img').attr('src')
     // $('div.image').slice(0,3).find('a').attr('href').split('?')[0]
 
-    list = list.get();
-
     // 处理所有的页面
-    const result = await ProcessFeed(list, ctx.cache);
+    const result = await ProcessFeed(data, ctx);
 
     return {
         title: '《新闻1+1》- 中央电视台新闻频道',

package/lib/v2/cctv/xwlb.js

@@ -0,0 +1,50 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const cheerio = require('cheerio');
+const dayjs = require('dayjs');
+const customParseFormat = require('dayjs/plugin/customParseFormat');
+dayjs.extend(customParseFormat);
+
+module.exports = async (ctx) => {
+    const res = await got({ method: 'get', url: 'https://tv.cctv.com/lm/xwlb/' });
+    const $ = cheerio.load(res.data);
+    // 解析最新一期新闻联播的日期
+    const latestDate = dayjs($('.rilititle p').text(), 'YYYY-MM-DD');
+    const count = [];
+    for (let i = 0; i < 20; i++) {
+        count.push(i);
+    }
+    const resultItems = await Promise.all(
+        count.map(async (i) => {
+            const newsDate = latestDate.subtract(i, 'days').hour(19);
+            const url = `https://tv.cctv.com/lm/xwlb/day/${newsDate.format('YYYYMMDD')}.shtml`;
+            const item = {
+                title: `新闻联播 ${newsDate.format('YYYY/MM/DD')}`,
+                link: url,
+                pubDate: timezone(parseDate(newsDate), +8),
+                description: await ctx.cache.tryGet(url, async () => {
+                    const res = await got(url);
+                    const content = cheerio.load(res.data);
+                    const list = new Array();
+                    content('body li').map((i, e) => {
+                        e = content(e);
+                        const href = e.find('a').attr('href');
+                        const title = e.find('a').attr('title');
+                        const dur = e.find('span').text();
+                        list.push(`<a href="${href}">${title} ⏱${dur}</a>`);
+                        return i;
+                    });
+                    return list.join('<br/>\n');
+                }),
+            };
+            return item;
+        })
+    );
+
+    ctx.state.data = {
+        title: 'CCTV 新闻联播',
+        link: 'http://tv.cctv.com/lm/xwlb/',
+        item: resultItems,
+    };
+};

package/lib/v2/chinacef/experts.js

@@ -0,0 +1,38 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const utils = require('./utils');
+
+module.exports = async (ctx) => {
+    const response = await got(`http://www.chinacef.cn/index.php/experts/zjmain/experts_id/${ctx.params.experts_id}`);
+
+    const data = response.data;
+
+    const $ = cheerio.load(data);
+
+    const domArray = $('div[class^="leftnews"]');
+    const rssTitle = $('title').text().trim();
+
+    const itemArray = await Promise.all(
+        domArray
+            .filter((index, item) => undefined !== $(item).find('h2 > a').attr('href'))
+            .map(async (index, item) => {
+                item = $(item);
+                const itemLink = utils.siteLink + item.find('h2 > a').attr('href');
+                const detail = await ctx.cache.tryGet(itemLink, async () => await utils.getArticleDetail(itemLink));
+
+                return {
+                    title: detail.title,
+                    description: detail.description,
+                    link: itemLink,
+                    pubDate: detail.time,
+                    author: detail.author,
+                };
+            })
+    );
+
+    ctx.state.data = {
+        title: rssTitle,
+        link: utils.siteLink,
+        item: itemArray,
+    };
+};

package/lib/v2/chinacef/hot.js

@@ -0,0 +1,38 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const utils = require('./utils');
+
+module.exports = async (ctx) => {
+    const response = await got.get('http://www.chinacef.cn/index.php/index/index');
+
+    const data = response.data;
+
+    const $ = cheerio.load(data);
+
+    const domArray = $('#boardright ul');
+    const rssTitle = '金融热点 - 首席经济学家论坛';
+
+    const itemArray = await Promise.all(
+        domArray
+            .map(async (index, item) => {
+                item = $(item);
+                const itemLink = utils.siteLink + item.find('li a').attr('href');
+                const detail = await ctx.cache.tryGet(itemLink, async () => await utils.getArticleDetail(itemLink));
+
+                return {
+                    title: detail.title,
+                    description: detail.description,
+                    link: itemLink,
+                    pubDate: detail.time,
+                    author: detail.author,
+                };
+            })
+            .get()
+    );
+
+    ctx.state.data = {
+        title: rssTitle,
+        link: utils.siteLink,
+        item: itemArray,
+    };
+};

package/lib/v2/chinacef/index.js

@@ -0,0 +1,39 @@
+const utils = require('./utils');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+
+module.exports = async (ctx) => {
+    const rssTitle = '最新更新 - 首席经济学家论坛';
+    const regex = /^\/index\.php\/index\/article\/article_id\/\d+/g;
+
+    const response = await got.get('http://www.chinacef.cn/index.php/index/index');
+    const $ = cheerio.load(response.data);
+
+    const articlesLinkListNode = $('a[target=_black]')
+        .filter((_, item) => item.attribs.href.match(regex))
+        .map((_, item) => item.attribs.href);
+    const articlesLinkList = Array.from(new Set(articlesLinkListNode));
+
+    const articles = await Promise.all(
+        articlesLinkList.map(async (item) => {
+            const articlesLink = utils.siteLink + item;
+
+            const detail = await ctx.cache.tryGet(articlesLink, async () => await utils.getArticleDetail(articlesLink));
+
+            const element = {
+                title: detail.title,
+                author: detail.author,
+                pubDate: detail.time,
+                description: detail.description,
+                link: articlesLink,
+            };
+            return element;
+        })
+    );
+
+    ctx.state.data = {
+        title: rssTitle,
+        link: utils.siteLink,
+        item: articles,
+    };
+};

package/lib/v2/chinacef/maintainer.js

@@ -0,0 +1,5 @@
+module.exports = {
+    '/': ['FledgeXu'],
+    '/:experts_id': ['kdanfly'],
+    '/portal/hot': ['kdanfly'],
+};

package/lib/v2/chinacef/radar.js

@@ -0,0 +1,25 @@
+module.exports = {
+    'chinacef.cn': {
+        _name: '首席经济学家论坛',
+        '.': [
+            {
+                title: '最新文章列表',
+                docs: 'https://docs.rsshub.app/finance.html#shou-xi-jing-ji-xue-jia-lun-tan',
+                source: ['/'],
+                target: '/chinacef',
+            },
+            {
+                title: '专家文章',
+                docs: 'https://docs.rsshub.app/finance.html#shou-xi-jing-ji-xue-jia-lun-tan-zhuan-jia',
+                source: ['/index.php/experts/zjmain/experts_id/:experts_id'],
+                target: '/chinacef/:experts_id',
+            },
+            {
+                title: '金融热点',
+                docs: 'https://docs.rsshub.app/finance.html#shou-xi-jing-ji-xue-jia-lun-tan-jin-rong-re-dian',
+                source: ['/index.php/index/index'],
+                target: '/chinacef/portal/hot',
+            },
+        ],
+    },
+};

package/lib/v2/chinacef/router.js

@@ -0,0 +1,5 @@
+module.exports = function (router) {
+    router.get('/:experts_id', require('./experts'));
+    router.get('/portal/hot', require('./hot'));
+    router.get('/', require('./index'));
+};

package/lib/v2/chinacef/utils.js

@@ -0,0 +1,55 @@
+const { art } = require('@/utils/render');
+const path = require('path');
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const siteLink = 'http://www.chinacef.cn';
+
+const renderDesc = (desc) =>
+    art(path.join(__dirname, 'templates/description.art'), {
+        desc,
+    });
+
+const cleanDom = (dom) => {
+    dom('*[style]').removeAttr('style');
+    dom('br').remove();
+    dom('span:empty').remove();
+    dom('p:empty').remove();
+    dom('p').each((_, el) => {
+        if (dom(el).html().trim() === '') {
+            dom(el).remove();
+        }
+    });
+    return dom;
+};
+
+const getArticleDetail = async (link) => {
+    const response = await got({
+        method: 'get',
+        url: link,
+    });
+    const $ = cleanDom(cheerio.load(response.data));
+
+    const title = $('span[class=contenttitle]').first().text().trim();
+    const author = $('a[rel=author]').first().text().trim();
+    const time = parseDate($('.divwidth').text().trim().split(' ')[3], 'YYYY-MM-DD');
+    const description = $('div[class=newsmaintext]').html();
+    return new ArticleDetail(title, author, time, renderDesc(description));
+};
+class ArticleDetail {
+    constructor(title, author, time, description) {
+        this.title = title;
+        this.author = author;
+        this.time = time;
+        this.description = description;
+    }
+}
+
+module.exports = {
+    siteLink,
+    renderDesc,
+    cleanDom,
+    getArticleDetail,
+    ArticleDetail,
+};

package/lib/v2/chinanews/index.js

@@ -0,0 +1,65 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const cheerio = require('cheerio');
+
+const rootUrl = 'https://www.chinanews.com.cn';
+
+module.exports = async (ctx) => {
+    const currentUrl = `${rootUrl}/scroll-news/news1.html`;
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+    const $ = cheerio.load(response.data);
+    const list = $('a', '.dd_bt')
+        .map((_, item) => ({
+            link: rootUrl + $(item).attr('href'),
+            title: $(item).text(),
+        }))
+        .get()
+        .slice(0, ctx.query.limit ? (parseInt(ctx.query.limit) > 125 ? 125 : parseInt(ctx.query.limit)) : 50);
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(detailResponse.data);
+                if (content('div.content_desc').length > 0) {
+                    item.description = content('div.content_desc').text();
+                    item.author = content('a.source').text();
+                    const info = content('p', '.left').text().trim().slice(5).split(' ');
+                    item.author = info[2].trim();
+                    item.pubDate = timezone(parseDate(info[0] + info[1], 'YYYY年MM月DD日HH:mm'), +8);
+                } else if (content('div.t3').length > 0) {
+                    item.description = content('div.t3').text();
+                    const info = content('div[style="text-align:right;font-size:12px;"]').text().slice(5).split(' ');
+                    item.author = info[2];
+                    item.pubDate = timezone(parseDate(info[0] + info[1], 'YYYY-MM-DDHH:mm'), +8);
+                } else {
+                    item.description = content('div.left_zw').text();
+                    const info = content('div.left-t')
+                        .contents()
+                        .filter(function () {
+                            return this.type === 'text';
+                        })
+                        .text()
+                        .split('　');
+                    item.pubDate = timezone(parseDate(info[0], 'YYYY年MM月DD日 HH:mm'), +8);
+                    item.author = info[1] + content('a.source').text();
+                }
+                return item;
+            })
+        )
+    );
+    ctx.state.data = {
+        title: '中国新闻网',
+        link: currentUrl,
+        description: '中国新闻网（简称“中新网”），由中国新闻社主办，为中央重点新闻网站。',
+        language: 'zh-cn',
+        item: items,
+    };
+};

package/lib/v2/chinanews/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/': ['yuxinliu-alex'],
+};

package/lib/v2/chinanews/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'chinanews.com.cn': {
+        _name: '中国新闻网',
+        '.': [
+            {
+                title: '最新',
+                docs: 'https://docs.rsshub.app/traditional-media.html#zhong-xin-wang',
+                source: ['/'],
+                target: '/chinanews',
+            },
+        ],
+    },
+};

package/lib/v2/chinanews/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/', require('./index'));
+};

package/lib/v2/cnbeta/index.js

@@ -5,7 +5,8 @@ const { rootUrl, ProcessItems } = require('./utils');
 module.exports = async (ctx) => {
     const currentUrl = `${rootUrl}/backend.php`;
 
-    const { items } = await parser.parseURL(currentUrl);
+    let { items } = await parser.parseURL(currentUrl);
+    items = items.filter((item) => item.link.includes('cnbeta'));
 
     ctx.state.data = {
         title: 'cnBeta',

package/lib/v2/cnbeta/utils.js

@@ -19,6 +19,7 @@ module.exports = {
                     content('.topic, .article-topic, .article-global').remove();
 
                     item.description = content('.article-summary').html() + content('.article-content').html();
+                    item.author = content('header.title div.meta span.source').text();
 
                     return item;
                 })

package/lib/v2/dapenti/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/tugua': ['tgly307'],
+    '/subject/:id': ['xyqfer'],
+};

package/lib/v2/dapenti/radar.js

@@ -0,0 +1,27 @@
+module.exports = {
+    'dapenti.com': {
+        _name: '喷嚏',
+        '.': [
+            {
+                title: '图卦',
+                docs: 'https://docs.rsshub.app/picture.html#pen-ti',
+                source: ['/blog/blog.asp'],
+                target: (params, url) => {
+                    if (new URL(url).searchParams.get('subjectid') === '70') {
+                        return '/dapenti/tugua';
+                    }
+                },
+            },
+            {
+                title: '主题',
+                docs: 'https://docs.rsshub.app/picture.html#pen-ti',
+                source: ['/blog/blog.asp'],
+                target: (params, url) => {
+                    if (new URL(url).searchParams.get('subjectid')) {
+                        return '/dapenti/subject/' + new URL(url).searchParams.get('subjectid');
+                    }
+                },
+            },
+        ],
+    },
+};

package/lib/v2/dapenti/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/tugua', require('./tugua'));
+    router.get('/subject/:id', require('./subject'));
+};

package/lib/{routes → v2}/dapenti/subject.js

@@ -1,5 +1,5 @@
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
-    ctx.state.data = await utils.parseFeed({ subjectid: ctx.params.id });
+    ctx.state.data = await utils.parseFeed({ subjectid: ctx.params.id }, ctx);
 };

package/lib/{routes → v2}/dapenti/tugua.js

@@ -1,5 +1,5 @@
 const utils = require('./utils');
 
 module.exports = async (ctx) => {
-    ctx.state.data = await utils.parseFeed({ subjectid: 70 });
+    ctx.state.data = await utils.parseFeed({ subjectid: 70 }, ctx);
 };