rsshub 1.0.0-master.fe4f650 → 1.0.0-master.fe560d2

package/lib/v2/curius/templates/description.art

@@ -0,0 +1,16 @@
+{{ if item.metadata.full_text }}
+    原文：{{@ item.metadata.full_text.replace(/\n/gm, '<br>') }}<br><br>
+{{ /if }}
+{{ if item.comments.length }}
+    评论：{{ item.comments[0].text.substring(0, 100) }}<br><br>
+{{ /if }}
+{{ if item.highlights.length }}
+    评论：
+    {{ each item.highlights highlight }}
+        {{ if highlight.comment }}
+            {{ highlight.highlight }}<br>评论：{{ highlight.comment.text }}<br>
+        {{ else }}
+            <br>标注：{{ highlight.highlight }}<br>
+        {{ /if }}
+    {{ /each }}
+{{ /if }}

package/lib/v2/douyin/hashtag.js

@@ -1,4 +1,3 @@
-const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const config = require('@/config').value;
@@ -23,19 +22,26 @@ module.exports = async (ctx) => {
             const browser = await require('@/utils/puppeteer')();
             const page = await browser.newPage();
             await page.setRequestInterception(true);
+            let awemeList = '';
             page.on('request', (request) => {
-                request.resourceType() === 'document' ? request.continue() : request.abort();
+                request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+            });
+            page.on('response', async (response) => {
+                const request = response.request();
+                if (request.url().includes('/v1/web/challenge/aweme')) {
+                    awemeList = await response.json();
+                }
             });
             await page.goto(tagUrl, {
-                waitUntil: 'domcontentloaded',
+                waitUntil: 'networkidle2',
             });
             await page.waitForSelector('#RENDER_DATA');
-            const html = await page.evaluate(() => document.documentElement.innerHTML);
+            const html = await page.evaluate(() => document.getElementById('RENDER_DATA').innerText);
             await browser.close();
 
-            const $ = cheerio.load(html);
-            const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').text()));
-            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail && renderData[key].defaultData);
+            const renderData = JSON.parse(decodeURIComponent(html));
+            const dataKey = Object.keys(renderData).find((key) => renderData[key].topicDetail);
+            renderData[dataKey].defaultData = awemeList.aweme_list;
             return renderData[dataKey];
         },
         config.cache.routeExpire,
@@ -49,7 +55,7 @@ module.exports = async (ctx) => {
     const posts = tagData.defaultData;
     const items = posts.map((post) => {
         // parse video
-        let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
+        let videoList = post.video && post.video.bit_rate && post.video.bit_rate.map((bit_rate) => resolveUrl(bit_rate.play_addr.url_list[0]));
         if (relay) {
             videoList = videoList.map((item) => proxyVideo(item, relay));
         }
@@ -57,14 +63,9 @@ module.exports = async (ctx) => {
         duration = duration && duration / 1000;
         let img;
         // if (!embed) {
-        //     img = post.video && post.video.dynamicCover; // dynamic cover (webp)
+        //     img = post.video && post.video.dynamic_cover && post.video.dynamic_cover.url_list[post.video.dynamic_cover.url_list.length - 1]; // dynamic cover (webp)
         // }
-        img =
-            img ||
-            (post.video &&
-                ((post.video.coverUrlList && post.video.coverUrlList[post.video.coverUrlList.length - 1]) || // HD
-                    post.video.originCover || // LD
-                    post.video.cover)); // even more LD
+        img = img || (post.video && post.video.origin_cover && post.video.origin_cover.url_list[post.video.origin_cover.url_list.length - 1]);
         img = img && resolveUrl(img);
 
         // render description
@@ -76,10 +77,10 @@ module.exports = async (ctx) => {
         return {
             title: post.desc,
             description,
-            link: `https://www.douyin.com/video/${post.awemeId}`,
-            pubDate: parseDate(post.createTime * 1000),
-            category: post.textExtra.map((extra) => extra.hashtagName),
-            author: post.authorInfo.nickname,
+            link: `https://www.douyin.com/video/${post.aweme_id}`,
+            pubDate: parseDate(post.create_time, 'X'),
+            category: post.text_extra.map((extra) => extra.hashtag_name),
+            author: post.author.nickname,
         };
     });
 

package/lib/v2/douyin/live.js

@@ -1,5 +1,5 @@
 const config = require('@/config').value;
-const { getOriginAvatar } = require('./utils');
+const { getOriginAvatar, universalGet } = require('./utils');
 
 module.exports = async (ctx) => {
     const { rid } = ctx.params;
@@ -9,26 +9,7 @@ module.exports = async (ctx) => {
 
     const pageUrl = `https://live.douyin.com/${rid}`;
 
-    const renderData = await ctx.cache.tryGet(
-        `douyin:live:${rid}`,
-        async () => {
-            const browser = await require('@/utils/puppeteer')();
-            const page = await browser.newPage();
-            await page.setRequestInterception(true);
-            page.on('request', (request) => {
-                request.resourceType() === 'document' ? request.continue() : request.abort();
-            });
-            await page.goto(pageUrl, {
-                waitUntil: 'domcontentloaded',
-            });
-            const data = JSON.parse(decodeURIComponent(await page.evaluate(() => document.querySelector('#RENDER_DATA').innerText)));
-            browser.close();
-
-            return data;
-        },
-        config.cache.routeExpire,
-        false
-    );
+    const renderData = await ctx.cache.tryGet(`douyin:live:${rid}`, () => universalGet(pageUrl, 'live', ctx), config.cache.routeExpire, false);
 
     if (renderData[Object.keys(renderData).find((k) => renderData[k].status_code)].status_code !== '0') {
         throw `Status code ${renderData[Object.keys(renderData).find((k) => renderData[k].status_code)].status_code}`;

package/lib/v2/douyin/user.js

@@ -1,9 +1,8 @@
-const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
 const { art } = require('@/utils/render');
 const config = require('@/config').value;
 const { fallback, queryToBoolean } = require('@/utils/readable-social');
-const { templates, resolveUrl, proxyVideo, getOriginAvatar } = require('./utils');
+const { templates, resolveUrl, proxyVideo, getOriginAvatar, universalGet } = require('./utils');
 
 module.exports = async (ctx) => {
     const uid = ctx.params.uid;
@@ -20,27 +19,13 @@ module.exports = async (ctx) => {
     const pageData = await ctx.cache.tryGet(
         `douyin:user:${uid}`,
         async () => {
-            const browser = await require('@/utils/puppeteer')();
-            const page = await browser.newPage();
-            await page.setRequestInterception(true);
-            page.on('request', (request) => {
-                request.resourceType() === 'document' ? request.continue() : request.abort();
-            });
-            await page.goto(pageUrl, {
-                waitUntil: 'domcontentloaded',
-            });
-            const html = await page.evaluate(() => document.documentElement.innerHTML);
-            await browser.close();
-
-            const $ = cheerio.load(html);
-            const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
+            const renderData = await universalGet(pageUrl, 'user', ctx);
             const dataKey = Object.keys(renderData).filter((key) => renderData[key].user && renderData[key].post)[0];
             return renderData[dataKey];
         },
         config.cache.routeExpire,
         false
     );
-
     const userInfo = pageData.user.user;
     const userNickName = userInfo.nickname;
     const userDescription = userInfo.desc;

package/lib/v2/douyin/utils.js

@@ -1,4 +1,8 @@
 const path = require('path');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const logger = require('@/utils/logger');
+const config = require('@/config').value;
 
 const templates = {
     desc: path.join(__dirname, 'templates/desc.art'),
@@ -42,9 +46,90 @@ const getOriginAvatar = (url) =>
         .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
         .replace(/~\w+_\d+x\d+/g, '');
 
+const extractRenderData = (html) => {
+    if (!html) {
+        throw Error('Empty html. The request may be filtered by WAF.');
+    }
+
+    const $ = cheerio.load(html);
+    const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
+    if (!renderData) {
+        const title = $('title').text();
+        if (title.includes('验证')) {
+            throw Error(title);
+        } else if ($('#captcha_container').length > 0) {
+            throw Error('Captcha required.');
+        }
+        throw Error('Failed to get render data.');
+    } else if (renderData.isSpider) {
+        throw Error('The request was considered to be from a spider.');
+    } else {
+        return renderData;
+    }
+};
+
+const gotGet = async (url, ua) => {
+    const response = await got(url, {
+        headers: {
+            'User-Agent': ua, // magic!
+        },
+    });
+    return extractRenderData(response.data);
+};
+
+const puppeteerGet = async (pageUrl) => {
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.setRequestInterception(true);
+    page.on('request', (request) => {
+        request.resourceType() === 'document' ? request.continue() : request.abort();
+    });
+    await page.goto(pageUrl, {
+        waitUntil: 'domcontentloaded',
+    });
+    const html = await page.evaluate(() => document.documentElement.innerHTML);
+    await browser.close();
+    return extractRenderData(html);
+};
+
+const universalGet = async (url, route, ctx) => {
+    const cacheKey = `douyin:utils:universalGet:gotFirst:${route}`;
+    const gotFirst = await ctx.cache.tryGet(
+        cacheKey,
+        () => '1',
+        config.cache.contentExpire,
+        false // no refresh now
+    );
+
+    const ua = route === 'live' ? 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' : 'curl/7.86.0';
+
+    let data;
+    if (gotFirst === '1') {
+        try {
+            data = await gotGet(url, ua);
+            ctx.cache.set(cacheKey, '1', config.cache.contentExpire); // refresh if success
+        } catch (e) {
+            logger.warn('Failed to get data with got, trying puppeteer: ' + e.message);
+            data = await puppeteerGet(url);
+            ctx.cache.set(cacheKey, '0', config.cache.contentExpire); // only set if success
+        }
+    } else {
+        try {
+            data = await puppeteerGet(url);
+            ctx.cache.set(cacheKey, '0', config.cache.contentExpire); // refresh if success
+        } catch (e) {
+            logger.warn('Failed to get data with puppeteer, trying got: ' + e.message);
+            data = await gotGet(url, ua);
+            ctx.cache.set(cacheKey, '1', config.cache.contentExpire); // only set if success
+        }
+    }
+    return data;
+};
+
 module.exports = {
     templates,
     resolveUrl,
     proxyVideo,
     getOriginAvatar,
+    universalGet,
 };

package/lib/v2/{acm-ecnu → ecnu}/contest.js

@@ -9,10 +9,7 @@ module.exports = async (ctx) => {
     const rootUrl = 'https://acm.ecnu.edu.cn';
     const currentUrl = `${rootUrl}/contest/`;
 
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
+    const response = await got(currentUrl);
 
     const $ = cheerio.load(response.data);
     const $trList = $('div > div > table > tbody > tr');

package/lib/v2/ecnu/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/acm/contest/:category?': ['a180285'],
+    '/yjs': ['shengmaosu'],
+};

package/lib/v2/ecnu/radar.js

@@ -0,0 +1,30 @@
+module.exports = {
+    'ecnu.edu.cn': {
+        _name: '华东师范大学',
+        acm: [
+            {
+                title: '所有比赛列表',
+                docs: 'https://docs.rsshub.app/university.html#hua-dong-shi-fan-da-xue',
+                source: ['/contest/', '/'],
+                target: '/ecnu/acm/contest/',
+            },
+            {
+                title: '仅公开比赛列表',
+                docs: 'https://docs.rsshub.app/university.html#hua-dong-shi-fan-da-xue',
+                source: ['/contest/', '/'],
+                target: '/ecnu/acm/contest/public',
+            },
+        ],
+    },
+    'kaoyan.com': {
+        _name: '华东师范大学',
+        yz: [
+            {
+                title: '研究生院',
+                docs: 'https://docs.rsshub.app/university.html#hua-dong-shi-fan-da-xue',
+                source: ['/ecnu/tiaoji', '/'],
+                target: '/ecnu/yjs',
+            },
+        ],
+    },
+};

package/lib/v2/ecnu/router.js

@@ -0,0 +1,4 @@
+module.exports = function (router) {
+    router.get('/acm/contest/:category?', require('./contest'));
+    router.get('/yjs', require('./yjs'));
+};

package/lib/v2/ecnu/yjs.js

@@ -0,0 +1,37 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+
+module.exports = async (ctx) => {
+    const link = 'https://yz.kaoyan.com/ecnu/tiaoji/';
+    const response = await got(link);
+    const $ = cheerio.load(response.data);
+    const list = $('.subList li')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('li a').text(),
+                link: item.find('li a').attr('href').replace('http:', 'https:'),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const { data: response } = await got(item.link);
+                const $ = cheerio.load(response);
+                item.description = $('.articleCon').html();
+                item.pubDate = parseDate($('.outer_utime').text());
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: '华东师范大学研究生院',
+        link,
+        description: '华东师范大学研究生调剂信息',
+        item: items,
+    };
+};

package/lib/v2/ehentai/ehapi.js

@@ -50,15 +50,27 @@ async function parsePage(cache, data, get_bittorrent = false, embed_thumb = fals
     // "l" for Compact
     // "e" for Extended
     // "t" for Thumbnail
-    const layout = $('div#dms option[selected]').attr('value');
-
-    let galleries;
-    if (layout !== 't') {
-        galleries = $('table[class^="itg glt"] tbody');
-    } else {
-        // Thumbnail layout has different page structure
-        // resolve it here
-        galleries = $('div[class^="itg gld"]');
+    let layout = 't';
+
+    // "itg gld" for Thumbnail
+    let galleries = $('div[class^="itg gld"]');
+    // "itg gltm" for Minimal or Minimal+
+    if (galleries.length <= 0) {
+        galleries = $('table[class^="itg gltm"] tbody');
+        layout = 'm';
+    }
+    // "itg gltc" for Compact
+    if (galleries.length <= 0) {
+        galleries = $('table[class^="itg gltc"] tbody');
+        layout = 'l';
+    }
+    // "itg glte" for Extended
+    if (galleries.length <= 0) {
+        galleries = $('table[class^="itg glte"] tbody');
+        layout = 'e';
+    }
+    if (galleries.length <= 0) {
+        return [];
     }
 
     async function parseElement(cache, element) {
@@ -108,12 +120,11 @@ async function parsePage(cache, data, get_bittorrent = false, embed_thumb = fals
             if ('le'.includes(layout)) {
                 // artist tags will only show in Compact or Extended layout
                 // get artist names as author
-                const artist = $(el)
+                item.author = $(el)
                     .find('div.gt[title^="artist:"]')
                     .toArray()
-                    .map((i, tag) => $(tag).text())
+                    .map((tag) => $(tag).text())
                     .join(' / ');
-                item.author = artist;
             }
             return item;
         }

package/lib/v2/foresightnews/index.js

@@ -1,6 +1,7 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
 const { parseDate } = require('@/utils/parse-date');
+const zlib = require('zlib');
 
 const categories = {
     '': {
@@ -28,12 +29,11 @@ module.exports = async (ctx) => {
     const rootUrl = 'https://foresightnews.pro';
     const currentUrl = `${rootUrl}/v1/${categories[category].id}?page=1&size=${ctx.query.limit ?? 30}${category === 'column' ? `&column_id=${id}` : ''}`;
 
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
+    const response = await got(currentUrl);
 
-    let items = response.data.data.list.map((item) =>
+    const list = JSON.parse(zlib.inflateSync(Buffer.from(response.data.data.list, 'base64')).toString());
+
+    let items = list.map((item) =>
         item.source_type
             ? {
                   type: item.source_type,
@@ -47,8 +47,8 @@ module.exports = async (ctx) => {
             : {
                   type: category,
                   title: item.title,
-                  author: item?.author?.username ?? undefined,
-                  link: `${rootUrl}/${category === 'article' ? 'article' : 'news'}/detail/${item.id}`,
+                  author: item.author?.username ?? undefined,
+                  link: `${rootUrl}/${category === 'article' || category === 'column' ? 'article' : 'news'}/detail/${item.id}`,
                   category: item.tags ? item.tags.map((tag) => tag.name) : undefined,
                   description: item.content ?? '',
                   pubDate: parseDate(item.publishDate),
@@ -63,10 +63,7 @@ module.exports = async (ctx) => {
         items.map((item) =>
             ctx.cache.tryGet(item.link, async () => {
                 if (item.type !== 'news') {
-                    const detailResponse = await got({
-                        method: 'get',
-                        url: item.link,
-                    });
+                    const detailResponse = await got(item.link);
 
                     const content = cheerio.load(detailResponse.data);
 

package/lib/v2/foreverblog/feeds.js

@@ -0,0 +1,36 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+
+module.exports = async (ctx) => {
+    const currentUrl = 'https://www.foreverblog.cn/feeds.html';
+
+    const response = await got(currentUrl);
+
+    const $ = cheerio.load(response.data);
+    const $articles = $('article[class="post post-type-normal"]');
+    const items = $articles
+        .map((_, el) => {
+            const $titleDiv = $(el).find('h1[class="post-title"]');
+            const title = $titleDiv.text().trim();
+            const link = $titleDiv.find('a').eq(0).attr('href');
+            const author = $(el).find('div[class="post-author"]').text().trim();
+            const postDate = $(el).find('time').text().trim();
+            const pubDate = timezone(parseDate(postDate, 'MM-DD'), +8);
+            const description = `${author}: ${title}`;
+            return {
+                title: description,
+                description,
+                link,
+                pubDate,
+            };
+        })
+        .toArray();
+
+    ctx.state.data = {
+        title: '十年之约——专题展示',
+        link: currentUrl,
+        item: items,
+    };
+};

package/lib/v2/foreverblog/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/feeds': ['7Wate', 'a180285'],
+};

package/lib/v2/foreverblog/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'foreverblog.cn': {
+        _name: '十年之约',
+        www: [
+            {
+                title: '专题展示',
+                docs: 'https://docs.rsshub.app/blog.html#shi-nian-zhi-yue',
+                source: ['/feeds.html'],
+                target: '/foreverblog/feeds',
+            },
+        ],
+    },
+};

package/lib/v2/foreverblog/router.js

@@ -0,0 +1,3 @@
+module.exports = function (router) {
+    router.get('/feeds', require('./feeds'));
+};

package/lib/v2/freewechat/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/profile/:id': ['TonyRL'],
+};

package/lib/v2/freewechat/profile.js

@@ -0,0 +1,61 @@
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+const { parseDate } = require('@/utils/parse-date');
+const timezone = require('@/utils/timezone');
+const { fixArticleContent } = require('@/utils/wechat-mp');
+const baseUrl = 'https://freewechat.com';
+
+module.exports = async (ctx) => {
+    const { id } = ctx.params;
+    const url = `${baseUrl}/profile/${id}`;
+    const { data: response } = await got(url);
+    const $ = cheerio.load(response);
+    const author = $('h2').text().trim();
+
+    const list = $('.main')
+        .toArray()
+        .slice(0, -1) // last item is a template
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('a').text(),
+                author,
+                link: `${baseUrl}${item.find('a').attr('href')}`,
+                description: item.find('.preview').text(),
+            };
+        });
+
+    const items = await Promise.all(
+        list.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const response = await got(item.link, {
+                    headers: {
+                        Referer: url,
+                    },
+                });
+                const $ = cheerio.load(response.data);
+
+                $('.js_img_placeholder').remove();
+                $('amp-img').each((_, e) => {
+                    e = $(e);
+                    e.replaceWith(`<img src="${new URL(e.attr('src'), response.url).href}" width="${e.attr('width')}" height="${e.attr('height')}" decoding="async">`);
+                });
+                $('amp-video').each((_, e) => {
+                    e = $(e);
+                    e.replaceWith(`<video width="${e.attr('width')}" height="${e.attr('height')}" controls poster="${e.attr('poster')}">${e.html()}</video>`);
+                });
+
+                item.description = fixArticleContent($('#js_content'));
+                item.pubDate = timezone(parseDate($('#publish_time').text()), +8);
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: $('head title').text(),
+        link: url,
+        image: 'https://freewechat.com/favicon.ico',
+        item: items,
+    };
+};

package/lib/v2/freewechat/radar.js

@@ -0,0 +1,13 @@
+module.exports = {
+    'freewechat.com': {
+        _name: '自由微信',
+        '.': [
+            {
+                title: '公众号',
+                docs: 'https://docs.rsshub.app/new-media.html#zi-you-wei-xin',
+                source: ['/profile/:id'],
+                target: '/freewechat/profile/:id',
+            },
+        ],
+    },
+};

package/lib/v2/freewechat/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/profile/:id', require('./profile'));
+};

package/lib/v2/gcores/category.js

@@ -12,16 +12,26 @@ module.exports = async (ctx) => {
     const $ = cheerio.load(data);
     const feedTitle = $('title').text();
 
-    const list = $('.original.am_card.original-normal')
-        .map(function () {
+    let list;
+    if (category === 'news') {
+        list = $('a.news').map(function () {
+            const item = {
+                url: $(this).attr('href'),
+                title: $(this).find('.news_content>h3').text(),
+            };
+            return item;
+        });
+    } else {
+        list = $('.original.am_card.original-normal').map(function () {
             const item = {
                 url: $(this).find('.am_card_inner>a').attr('href'),
                 title: $(this).find('h3.am_card_title').text(),
                 category: $(this).find('span.original_category>a').text(),
             };
             return item;
-        })
-        .get();
+        });
+    }
+    list = list.get();
 
     if (list.length > 0 && list.every((item) => item.url === undefined)) {
         throw new Error('Article URL not found! Please submit an issue on GitHub.');
@@ -85,13 +95,13 @@ module.exports = async (ctx) => {
                 $('.story_hidden').replaceWith('');
 
                 const content = $('.story.story-show').html();
-                return {
+                const basicItem = {
                     title: item.title,
                     description: cover + content,
-                    category: item.category,
                     link: articleUrl,
                     guid: articleUrl,
                 };
+                return category === 'news' ? basicItem : { ...basicItem, category: item.category };
             });
         })
     );