rsshub 1.0.0-master.f97cdb2 → 1.0.0-master.fa3428f

package/lib/utils/wechat-mp.js

@@ -2,19 +2,19 @@
  * Author: @Rongronggg9
  *
  * There are at least three folders which are relevant with WeChat MP (Official Account Platform / Media Platform):
- * lib/route/tencent/wechat
  * lib/v2/wechat
  * lib/v2/gzh360
+ * lib/v2/sdu/cs
  *
  * If your new route is not in the above folders, please add it to the list.
  *
- * If your route need to fetch MP articles from mp.weixin.qq.com, you SHOULD use `finishArticleItem`.
+ * If your route needs to fetch MP articles from mp.weixin.qq.com, you SHOULD use `finishArticleItem`.
  * However, if your route need to determine some metadata by itself, you MAY use `fetchArticle`.
  * If you find more metadata on the webpage, consider modifying `fetchArticle` to include them.
  * NEVER fetch MP articles from mp.weixin.qq.com in your route in order to avoid cache key collision.
  * NO NEED TO use cache if you are using `finishArticleItem` or `fetchArticle`, they will handle cache for you.
  *
- * If your route fetch MP articles from other websites, you SHOULD use `fixArticleContent` to fix the content format.
+ * If your route fetches MP articles from other websites, you SHOULD use `fixArticleContent` to fix the content format.
  * If you find more fixes that should be applied, consider modifying `fixArticleContent` to include them.
  *
  * For more details of these functions, please refer to the jsDoc in the source code.
@@ -35,6 +35,31 @@ const replaceTag = ($, oldTag, newTagName) => {
     oldTag.replaceWith(NewTag);
 };
 
+const detectOriginalArticleUrl = ($) => {
+    // No article content get, try the original url
+    // example: https://mp.weixin.qq.com/s/f6sKObaZZhADTYU2Jl5Bnw
+    if (!$('#js_content').text()) {
+        return $('#js_share_source').attr('data-url');
+    }
+    // Article content is too short, try the first link
+    // example: https://mp.weixin.qq.com/s/9saVB4KaolRyJfpajzeFRg
+    if ($('#js_content').text().length < 80) {
+        return $('#js_content a').attr('href');
+    }
+    return null;
+};
+
+const detectSourceUrl = ($) => {
+    const matchs = $.root()
+        .html()
+        .match(/msg_source_url = '(.+)';/);
+
+    if (matchs) {
+        return matchs[1];
+    }
+    return null;
+};
+
 /**
  * Articles from WeChat MP have weird formats, this function is used to fix them.
  *
@@ -77,11 +102,69 @@ const fixArticleContent = (html, skipImg = false) => {
             replaceTag($, section, 'div');
         }
     });
+
+    // fix single picture article
+    // example: https://mp.weixin.qq.com/s/4p5YmYuASiQSYFiy7KqydQ
+    $('script').each((_, script) => {
+        script = $(script);
+        const matchs = script.html().match(/document\.getElementById\('js_image_desc'\)\.innerHTML = "(.*)"\.replace/);
+
+        if (matchs) {
+            script.replaceWith(matchs[1].replace(/\r/g, '').replace(/\n/g, '<br>').replace(/\\x0d/g, '').replace(/\\x0a/g, '<br>'));
+        }
+    });
+
     // clean scripts
     $('script').remove();
     return $.html();
 };
 
+// Ref:
+// https://soaked.in/2020/08/wechat-platform-url/
+// Known params (permanent long link):
+// __biz (essential), mid (essential), idx (essential), sn (essential), chksm, mpshare, scene, ascene, subscene, srcid,
+// lang, sharer_sharetime, sharer_shareid, version, exportkey, pass_ticket, clicktime, enterid, devicetype, nettype,
+// abtest_cookie, wx_header
+// Known params (temporary link):
+// src, timestamp, ver, signature, new (unessential)
+const normalizeUrl = (url, bypassHostCheck = false) => {
+    const oriUrl = url;
+    const urlObj = new URL(url);
+    if (!bypassHostCheck && urlObj.host !== 'mp.weixin.qq.com') {
+        throw new Error('wechat-mp: URL host must be "mp.weixin.qq.com", but got ' + oriUrl);
+    }
+    urlObj.protocol = 'https:';
+    urlObj.hash = ''; // remove hash
+    if (urlObj.pathname.match(/^\/s\/.+/)) {
+        // a short link, just remove all the params
+        urlObj.search = '';
+    } else if (urlObj.pathname.match(/^\/s$/)) {
+        const biz = urlObj.searchParams.get('__biz');
+        const mid = urlObj.searchParams.get('mid') || urlObj.searchParams.get('appmsgid');
+        const idx = urlObj.searchParams.get('idx') || urlObj.searchParams.get('itemidx');
+        const sn = urlObj.searchParams.get('sn') || urlObj.searchParams.get('sign');
+        if (biz && mid && idx && sn) {
+            // a permanent long link, remove all unessential params
+            // no need to escape anything so no need to use `new URLSearchParams({...}).toString()`
+            urlObj.search = `?__biz=${biz}&mid=${mid}&idx=${idx}&sn=${sn}`;
+        } else {
+            const src = urlObj.searchParams.get('src');
+            const timestamp = urlObj.searchParams.get('timestamp');
+            const ver = urlObj.searchParams.get('ver');
+            const signature = urlObj.searchParams.get('signature');
+            if (src && timestamp && ver && signature) {
+                // a temporary link, remove all unessential params
+                urlObj.search = `?src=${src}&timestamp=${timestamp}&ver=${ver}&signature=${signature}`;
+            } else {
+                // unknown link, just let it go
+            }
+        }
+    } else {
+        // IDK what it is, just let it go
+    }
+    return urlObj.href;
+};
+
 /**
  * Fetch article and its metadata from WeChat MP (mp.weixin.qq.com).
  *
@@ -92,20 +175,30 @@ const fixArticleContent = (html, skipImg = false) => {
  * @return {Promise<object>} - An object containing the article and its metadata.
  */
 const fetchArticle = async (ctx, url, bypassHostCheck = false) => {
-    const oriUrl = url;
-    url = url.replace(/^http:\/\//, 'https://').replace(/#\w*$/, ''); // normalize url
-    if (!bypassHostCheck && !url.startsWith('https://mp.weixin.qq.com/')) {
-        throw new Error('wechat-mp: URL must start with https://mp.weixin.qq.com/ or http://mp.weixin.qq.com/, but got ' + oriUrl);
-    }
+    url = normalizeUrl(url, bypassHostCheck);
     return await ctx.cache.tryGet(url, async () => {
         const response = await got(url);
         const $ = cheerio.load(response.data);
 
-        const title = $('meta[property="og:title"]').attr('content');
+        const title = ($('meta[property="og:title"]').attr('content') || '').replace(/\\r/g, '').replace(/\\n/g, ' ');
         const author = $('meta[name=author]').attr('content');
         let summary = $('meta[name=description]').attr('content');
         summary = summary !== title ? summary : '';
-        const description = fixArticleContent($('div#js_content.rich_media_content'));
+        let description = fixArticleContent($('#js_content'));
+
+        // No article get or article is too short, try the original url
+        const originalUrl = detectOriginalArticleUrl($);
+        if (originalUrl) {
+            // try to fetch the description from the original article
+            const originalResponse = await got(normalizeUrl(originalUrl, bypassHostCheck));
+            const original$ = cheerio.load(originalResponse.data);
+            description += fixArticleContent(original$('#js_content'));
+        }
+
+        const sourceUrl = detectSourceUrl($);
+        if (sourceUrl) {
+            description += `<a href="${sourceUrl}">阅读原文</a>`;
+        }
 
         let pubDate;
         const publish_time_script = $('script[nonce][type="text/javascript"]:contains("var ct")').first().html();
@@ -161,4 +254,7 @@ module.exports = {
     fixArticleContent,
     fetchArticle,
     finishArticleItem, // a new route SHOULD use this function instead of manually calling the above functions
+    _internal: {
+        normalizeUrl, // for internal use only, exported for testing
+    },
 };

package/lib/v2/agirls/index.js

@@ -4,6 +4,9 @@ const { parseDate } = require('@/utils/parse-date');
 const timezone = require('@/utils/timezone');
 const path = require('path');
 const { art } = require('@/utils/render');
+const { CookieJar } = require('tough-cookie');
+
+const cookieJar = new CookieJar();
 
 module.exports = async (ctx) => {
     const baseUrl = 'https://agirls.aotter.net';
@@ -14,6 +17,7 @@ module.exports = async (ctx) => {
         headers: {
             Referer: baseUrl,
         },
+        cookieJar,
     });
 
     const $ = cheerio.load(response.data);
@@ -36,6 +40,7 @@ module.exports = async (ctx) => {
                     headers: {
                         Referer: baseUrl + '/posts/' + (category ? `/${category}` : ''),
                     },
+                    cookieJar,
                 });
                 const content = cheerio.load(detailResponse.data);
 
@@ -67,5 +72,6 @@ module.exports = async (ctx) => {
         description: $('head meta[name=description]').attr('content'),
         item: items,
         language: $('html').attr('lang'),
+        cookieJar,
     };
 };

package/lib/v2/bilibili/video.js

@@ -32,6 +32,7 @@ module.exports = async (ctx) => {
                 pubDate: new Date(item.created * 1000).toUTCString(),
                 link: item.created > utils.bvidTime && item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : `https://www.bilibili.com/video/av${item.aid}`,
                 author: name,
+                comments: item.comment,
             })),
     };
 };

package/lib/v2/bookfere/category.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const url = 'https://bookfere.com/category/' + ctx.params.category;
+    const response = await got({
+        method: 'get',
+        url,
+    });
+
+    const data = response.data;
+
+    const $ = cheerio.load(data);
+    const list = $('main div div section');
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        item:
+            list &&
+            list
+                .map((index, item) => {
+                    item = $(item);
+                    const date = item.find('time').attr('datetime');
+                    const pubDate = parseDate(date);
+                    return {
+                        title: item.find('h2 a').text(),
+                        link: item.find('h2 a').attr('href'),
+                        pubDate,
+                        description: item.find('p').text(),
+                    };
+                })
+                .get(),
+    };
+};

package/lib/v2/bookfere/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category': ['OdinZhang'],
+};

package/lib/v2/bookfere/radar.js

@@ -0,0 +1,11 @@
+module.exports = {
+    'bookfere.com': {
+        _name: '书伴',
+        '.': [
+            {
+                title: '分类',
+                docs: 'https://docs.rsshub.app/reading.html#shu-ban',
+            },
+        ],
+    },
+};

package/lib/v2/bookfere/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/:category', require('./category'));
+};

package/lib/v2/cankaoxiaoxi/index.js

@@ -0,0 +1,81 @@
+const got = require('@/utils/got');
+const timezone = require('@/utils/timezone');
+const { parseDate } = require('@/utils/parse-date');
+const cheerio = require('cheerio');
+
+const nodes = {
+    china_news: {
+        title: '中国',
+        url: 'https://china.cankaoxiaoxi.com/',
+    },
+    world_news: {
+        title: '国际',
+        url: 'https://world.cankaoxiaoxi.com/',
+    },
+    military_news: {
+        title: '军事',
+        url: 'https://mil.cankaoxiaoxi.com/',
+    },
+    taiwan_news: {
+        title: '台海',
+        url: 'https://tw.cankaoxiaoxi.com/',
+    },
+    finance_news: {
+        title: '财经',
+        url: 'https://finance.cankaoxiaoxi.com/',
+    },
+    technology_news: {
+        title: '科技',
+        url: 'https://science.cankaoxiaoxi.com/',
+    },
+    culture_news: {
+        title: '文化',
+        url: 'https://culture.cankaoxiaoxi.com/',
+    },
+};
+
+module.exports = async (ctx) => {
+    const category = ctx.params.category;
+
+    const currentUrl = nodes[category].url;
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = cheerio.load(response.data);
+
+    const list = $('a', '.toutiao')
+        .filter(function () {
+            return $(this).text() !== '';
+        })
+        .map((_, item) => ({
+            link: $(item).attr('href'),
+        }))
+        .get();
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+                const content = cheerio.load(detailResponse.data);
+                item.title = content('h1.articleHead').text();
+                item.pubDate = timezone(parseDate(content('span#pubtime_baidu').text()), +8);
+                item.author = content('span#source_baidu').text();
+                item.description = content('div.articleText').html();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: `${nodes[category].title} 参考消息`,
+        link: currentUrl,
+        description: '参考消息',
+        language: 'zh-cn',
+        item: items,
+    };
+};

package/lib/v2/cankaoxiaoxi/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/news/:category': ['yuxinliu-alex'],
+};

package/lib/v2/cankaoxiaoxi/radar.js

@@ -0,0 +1,61 @@
+module.exports = {
+    'cankaoxiaoxi.com': {
+        _name: '参考消息',
+        china: [
+            {
+                title: '中国新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/china_news',
+            },
+        ],
+        culture: [
+            {
+                title: '文化新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/culture_news',
+            },
+        ],
+        finance: [
+            {
+                title: '财经新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/finance_news',
+            },
+        ],
+        mil: [
+            {
+                title: '军事新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/military_news',
+            },
+        ],
+        science: [
+            {
+                title: '科技新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/technology_news',
+            },
+        ],
+        tw: [
+            {
+                title: '台海新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/taiwan_news',
+            },
+        ],
+        world: [
+            {
+                title: '国际新闻',
+                docs: 'https://docs.rsshub.app/traditional-media.html#can-kao-xiao-xi',
+                source: ['/'],
+                target: '/cankaoxiaoxi/news/world_news',
+            },
+        ],
+    },
+};

package/lib/v2/cankaoxiaoxi/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/news/:category', require('./index'));
+};

package/lib/{routes → v2}/cctv/category.js

@@ -9,7 +9,7 @@ module.exports = async (ctx) => {
     } else if (category === 'xinwen1j1') {
         // 新闻1+1
         const xinwen1j1 = require('./utils/xinwen1j1');
-        responseData = await xinwen1j1(category, ctx);
+        responseData = await xinwen1j1(ctx);
     } else {
         // 央视新闻
         const getNews = require('./utils/news');

package/lib/{routes → v2}/cctv/jx.js

@@ -1,5 +1,6 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const rootUrl = 'https://photo.cctv.com';
@@ -12,7 +13,7 @@ module.exports = async (ctx) => {
     const $ = cheerio.load(response.data);
 
     const list = $('.textr a')
-        .slice(0, 10)
+        .slice(0, ctx.query.limit ? parseInt(ctx.query.limit) : 10)
         .map((_, item) => {
             item = $(item);
             return {
@@ -30,6 +31,10 @@ module.exports = async (ctx) => {
                     url: item.link,
                 });
                 const content = cheerio.load(detailResponse.data);
+                const date = content('head')
+                    .html()
+                    .match(/publishDate ="(.*) ";/)[1];
+                item.pubDate = date ? parseDate(date, 'YYYYMMDDHHmmss') : null;
 
                 item.description = content('.tujitop').html();
 

package/lib/{routes → v2}/cctv/lm.js

@@ -4,7 +4,7 @@ const timezone = require('@/utils/timezone');
 const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
-    const id = ctx.params.id || 'xwzk';
+    const id = ctx.params.id ?? 'xwzk';
 
     const rootUrl = 'https://tv.cctv.com';
     const apiRootUrl = 'https://api.cntv.cn';

package/lib/v2/cctv/maintainer.js

@@ -0,0 +1,7 @@
+module.exports = {
+    '/lm/:id?': ['nczitzk'],
+    '/photo/jx': ['nczitzk'],
+    '/special/:id?': ['nczitzk'],
+    '/xwlb': ['zengxs'],
+    '/:category': ['idealclover', 'xyqfer'],
+};

package/lib/v2/cctv/radar.js

@@ -0,0 +1,41 @@
+module.exports = {
+    'cctv.com': {
+        _name: '央视新闻',
+        news: [
+            {
+                title: '新闻专题',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yang-shi-xin-wen',
+                source: ['/special/*id'],
+                target: (params) => `/cctv/special/${params.id.split('/')[0]}`,
+            },
+            {
+                title: '专题',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yang-shi-xin-wen',
+                source: ['/:category'],
+                target: '/cctv/:category',
+            },
+        ],
+        photo: [
+            {
+                title: '央视网图片《镜象》',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yang-shi-xin-wen',
+                source: ['/jx', '/'],
+                target: '/cctv/photo/jx',
+            },
+        ],
+        tv: [
+            {
+                title: '新闻联播',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yang-shi-xin-wen',
+                source: ['/lm/xwlb', '/'],
+                target: '/cctv/xwlb',
+            },
+            {
+                title: '栏目',
+                docs: 'https://docs.rsshub.app/traditional-media.html#yang-shi-xin-wen',
+                source: ['/lm/*path', '/'],
+                target: (params) => `/cctv/lm/${params.path.replace('/videoset', '')}`,
+            },
+        ],
+    },
+};

package/lib/v2/cctv/router.js

@@ -0,0 +1,7 @@
+module.exports = (router) => {
+    router.get('/lm/:id?', require('./lm'));
+    router.get('/photo/jx', require('./jx'));
+    router.get('/special/:id?', require('./special'));
+    router.get('/xwlb', require('./xwlb'));
+    router.get('/:category', require('./category'));
+};

package/lib/{routes → v2}/cctv/special.js

@@ -1,8 +1,10 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async (ctx) => {
-    ctx.params.id = ctx.params.id || '315';
+    ctx.params.id = ctx.params.id ?? '315';
 
     let rootUrl;
 
@@ -75,10 +77,10 @@ module.exports = async (ctx) => {
                             dateSpan = content('span.time').text();
                         }
 
-                        date = dateSpan.replace(/年/g, '-').replace(/月/g, '-').replace(/日/g, '');
+                        date = timezone(parseDate(dateSpan, 'YYYY年MM月DD日 HH:mm:ss'), +8);
                     } else {
                         const dateString = match[1];
-                        date = `${dateString.slice(0, 4)}-${dateString.slice(4, 6)}-${dateString.slice(6, 8)} ${dateString.slice(8, 10)}:${dateString.slice(10, 12)}:${dateString.slice(12, 14)}`;
+                        date = timezone(parseDate(dateString, 'YYYYMMDDHHmmss'), +8);
                     }
 
                     if (content('#content_area').html() !== null) {
@@ -93,7 +95,7 @@ module.exports = async (ctx) => {
                     }
                 }
 
-                item.pubDate = new Date(date + ' GMT+8').toUTCString();
+                item.pubDate = date;
 
                 return item;
             })

package/lib/{routes → v2}/cctv/utils/mzzlbg.js

@@ -1,7 +1,9 @@
 const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
 
 module.exports = async () => {
-    const url = 'http://api.cntv.cn/video/videolistById?serviceId=cbox&vsid=C10354&em=01&p=1&n=50';
+    const url = 'https://api.cntv.cn/video/videolistById?serviceId=cbox&vsid=C10354&em=01&p=1&n=50';
 
     const response = await got({
         method: 'get',
@@ -12,26 +14,26 @@ module.exports = async () => {
 
     const resultItem = await Promise.all(
         data.video.map(async (video) => {
-            const url = `http://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid=${video.vid}`;
+            const url = `https://vdn.apps.cntv.cn/api/getHttpVideoInfo.do?pid=${video.vid}`;
             const item = {
                 title: video.t,
-                description: '',
+                description: video.desc,
                 link: video.url,
-                pubDate: new Date(video.ptime).toUTCString(),
+                pubDate: timezone(parseDate(video.ptime), +8),
             };
             const { data: videoDetail } = await got({
                 method: 'get',
                 url,
             });
 
-            item.description = `<video src="${videoDetail.hls_url}" controls="controls" poster="${video.img.replace(/\?.+/g, '')}" style="width: 100%"></video>`;
+            item.description += `<br><video src="${videoDetail.hls_url}" controls="controls" poster="${video.img.replace(/\?.+/g, '')}" style="width: 100%"></video>`;
             return item;
         })
     );
 
     return {
         title: '每周质量报告',
-        link: 'http://tv.cctv.com/lm/mzzlbg/videoset/index.shtml',
+        link: 'https://tv.cctv.com/lm/mzzlbg/videoset/index.shtml',
         description:
             '《每周质量报告》是CCTV-新闻频道一档以消费者为核心收视人群的新闻专题栏目。创办于2003年，始终致力于产品质量和食品安全领域的调查报道，以打假除劣扶优，推动质量进步为第一诉求，是我国电视新闻界质量新闻领域的旗帜性节目。首播时间：CCTV-新闻周日12:35—12:55。',
         item: resultItem,