rsshub 1.0.0-master.ff487f3 → 1.0.0-master.ff76aa7

package/lib/routes/douyin/user.ts

@@ -4,8 +4,11 @@ import { parseDate } from '@/utils/parse-date';
 import { art } from '@/utils/render';
 import { config } from '@/config';
 import { fallback, queryToBoolean } from '@/utils/readable-social';
-import { templates, resolveUrl, proxyVideo, getOriginAvatar, universalGet } from './utils';
+import { templates, resolveUrl, proxyVideo, getOriginAvatar } from './utils';
 import InvalidParameterError from '@/errors/types/invalid-parameter';
+import puppeteer from '@/utils/puppeteer';
+import logger from '@/utils/logger';
+import { PostData } from './types';
 
 export const route: Route = {
     path: '/user/:uid/:routeParams?',
@@ -43,29 +46,56 @@ async function handler(ctx) {
 
     const pageUrl = `https://www.douyin.com/user/${uid}`;
 
-    const pageData = await cache.tryGet(
+    const pageData = (await cache.tryGet(
         `douyin:user:${uid}`,
         async () => {
-            const renderData = await universalGet(pageUrl, 'user');
-            const dataKey = Object.keys(renderData).find((key) => renderData[key].user && renderData[key].post);
-            return renderData[dataKey];
+            let postData;
+            const browser = await puppeteer();
+            const page = await browser.newPage();
+            await page.setRequestInterception(true);
+
+            page.on('request', (request) => {
+                request.resourceType() === 'document' || request.resourceType() === 'script' || request.resourceType() === 'xhr' ? request.continue() : request.abort();
+            });
+            page.on('response', async (response) => {
+                const request = response.request();
+                if (request.url().includes('/web/aweme/post') && !postData) {
+                    postData = await response.json();
+                }
+            });
+
+            logger.http(`Requesting ${pageUrl}`);
+            await page.goto(pageUrl, {
+                waitUntil: 'networkidle2',
+            });
+
+            browser.close();
+
+            if (!postData) {
+                throw new Error('Empty post data. The request may be filtered by WAF.');
+            }
+
+            return postData;
         },
         config.cache.routeExpire,
         false
-    );
-    const userInfo = pageData.user.user;
+    )) as PostData;
+
+    if (!pageData.aweme_list?.length) {
+        throw new Error('Empty post data. The request may be filtered by WAF.');
+    }
+    const userInfo = pageData.aweme_list[0].author;
     const userNickName = userInfo.nickname;
-    const userDescription = userInfo.desc;
-    const userAvatar = getOriginAvatar(userInfo.avatar300Url || userInfo.avatarUrl);
+    // const userDescription = userInfo.desc;
+    const userAvatar = getOriginAvatar(userInfo.avatar_thumb.url_list.at(-1));
 
-    const posts = pageData.post.data;
-    const items = posts.map((post) => {
+    const items = pageData.aweme_list.map((post) => {
         // parse video
-        let videoList = post.video && post.video.bitRateList && post.video.bitRateList.map((item) => resolveUrl(item.playApi));
+        let videoList = post.video?.bit_rate?.map((item) => resolveUrl(item.play_addr.url_list.at(-1)));
         if (relay) {
             videoList = videoList.map((item) => proxyVideo(item, relay));
         }
-        let duration = post.video && post.video.duration;
+        let duration = post.video?.duration;
         duration = duration && duration / 1000;
         let img;
         // if (!embed) {
@@ -73,30 +103,28 @@ async function handler(ctx) {
         // }
         img =
             img ||
-            (post.video &&
-                ((post.video.coverUrlList && post.video.coverUrlList.at(-1)) || // HD
-                    post.video.originCover || // LD
-                    post.video.cover)); // even more LD
+            post.video?.cover?.url_list.at(-1) || // HD
+            post.video?.origin_cover?.url_list.at(-1); // LD
         img = img && resolveUrl(img);
 
         // render description
-        const desc = post.desc && post.desc.replaceAll('\n', '<br>');
+        const desc = post.desc?.replaceAll('\n', '<br>');
         let media = art(embed && videoList ? templates.embed : templates.cover, { img, videoList, duration });
         media = embed && videoList && iframe ? art(templates.iframe, { content: media }) : media; // warp in iframe
         const description = art(templates.desc, { desc, media });
 
         return {
-            title: post.desc,
+            title: post.desc.split('\n')[0],
             description,
-            link: `https://www.douyin.com/video/${post.awemeId}`,
-            pubDate: parseDate(post.createTime * 1000),
-            category: post.textExtra.map((extra) => extra.hashtagName),
+            link: `https://www.douyin.com/video/${post.aweme_id}`,
+            pubDate: parseDate(post.create_time * 1000),
+            category: post.video_tag.map((t) => t.tag_name),
         };
     });
 
     return {
         title: userNickName,
-        description: userDescription,
+        // description: userDescription,
         image: userAvatar,
         link: pageUrl,
         item: items,

package/lib/routes/douyin/utils.ts

@@ -1,13 +1,7 @@
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
-import cache from '@/utils/cache';
 import path from 'node:path';
-import got from '@/utils/got';
-import { load } from 'cheerio';
-import logger from '@/utils/logger';
-import { config } from '@/config';
-import puppeteer from '@/utils/puppeteer';
 
 const templates = {
     desc: path.join(__dirname, 'templates/desc.art'),
@@ -51,84 +45,4 @@ const getOriginAvatar = (url) =>
         .replace(/^(.*\.douyinpic\.com\/).*(\/aweme-avatar\/)([^?]*)(\?.*)?$/, '$1origin$2$3')
         .replaceAll(/~\w+_\d+x\d+/g, '');
 
-const extractRenderData = (html) => {
-    if (!html) {
-        throw new Error('Empty html. The request may be filtered by WAF.');
-    }
-
-    const $ = load(html);
-    const renderData = JSON.parse(decodeURIComponent($('script#RENDER_DATA').html()));
-    if (!renderData) {
-        const title = $('title').text();
-        if (title.includes('验证')) {
-            throw new Error(title);
-        } else if ($('#captcha_container').length > 0) {
-            throw new Error('Captcha required.');
-        }
-        throw new Error('Failed to get render data.');
-    } else if (renderData.isSpider) {
-        throw new Error('The request was considered to be from a spider.');
-    } else {
-        return renderData;
-    }
-};
-
-const gotGet = async (url, ua) => {
-    const response = await got(url, {
-        headers: {
-            'User-Agent': ua, // magic!
-        },
-    });
-    return extractRenderData(response.data);
-};
-
-const puppeteerGet = async (pageUrl) => {
-    const browser = await puppeteer();
-    const page = await browser.newPage();
-    await page.setRequestInterception(true);
-    page.on('request', (request) => {
-        request.resourceType() === 'document' ? request.continue() : request.abort();
-    });
-    await page.goto(pageUrl, {
-        waitUntil: 'domcontentloaded',
-    });
-    const html = await page.evaluate(() => document.documentElement.innerHTML);
-    await browser.close();
-    return extractRenderData(html);
-};
-
-const universalGet = async (url, route) => {
-    const cacheKey = `douyin:utils:universalGet:gotFirst:${route}`;
-    const gotFirst = await cache.tryGet(
-        cacheKey,
-        () => '1',
-        config.cache.contentExpire,
-        false // no refresh now
-    );
-
-    const ua = route === 'live' ? 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' : 'curl/7.86.0';
-
-    let data;
-    if (gotFirst === '1') {
-        try {
-            data = await gotGet(url, ua);
-            cache.set(cacheKey, '1', config.cache.contentExpire); // refresh if success
-        } catch (error) {
-            logger.warn('Failed to get data with got, trying puppeteer: ' + error.message);
-            data = await puppeteerGet(url);
-            cache.set(cacheKey, '0', config.cache.contentExpire); // only set if success
-        }
-    } else {
-        try {
-            data = await puppeteerGet(url);
-            cache.set(cacheKey, '0', config.cache.contentExpire); // refresh if success
-        } catch (error) {
-            logger.warn('Failed to get data with puppeteer, trying got: ' + error.message);
-            data = await gotGet(url, ua);
-            cache.set(cacheKey, '1', config.cache.contentExpire); // only set if success
-        }
-    }
-    return data;
-};
-
-export { templates, resolveUrl, proxyVideo, getOriginAvatar, universalGet };
+export { templates, resolveUrl, proxyVideo, getOriginAvatar };

package/lib/routes/dykszx/namespace.ts

@@ -0,0 +1,8 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '德阳人事考试网',
+    url: 'www.dykszx.com',
+    categories: ['government'],
+    description: '德阳人事考试网',
+};

package/lib/routes/dykszx/news.ts

@@ -0,0 +1,88 @@
+import { Route } from '@/types';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+import got from '@/utils/got';
+import cache from '@/utils/cache';
+
+const HOST = 'https://www.dykszx.com';
+
+const getContent = async (href) => {
+    const newsPage = `${HOST}${href}`;
+    const response = await got.get(newsPage);
+    const $ = load(response.data);
+    const newsTime =
+        $('body > div:nth-child(3) > div.page.w > div.shuxing.w')
+            .text()
+            .trim()
+            .match(/时间：(.*?)点击/g)?.[0] || '';
+    // 移除二维码
+    $('.sjlook').remove();
+    const content = $('#show-body').html() || '';
+    return { newsTime, content, newsPage };
+};
+
+const newsTypeObj = {
+    all: { selector: '#nrs > li > b', name: '新闻中心' },
+    gwy: { selector: 'body > div:nth-child(3) > div:nth-child(8) > ul > li', name: '公务员考试' },
+    sydw: { selector: 'body > div:nth-child(3) > div:nth-child(9) > ul > li', name: '事业单位考试' },
+    zyzc: { selector: 'body > div:nth-child(3) > div:nth-child(10) > ul > li', name: '执（职）业资格、职称考试' },
+    other: { selector: 'body > div:nth-child(3) > div:nth-child(11) > ul > li', name: '其他考试' },
+};
+
+async function handler(ctx) {
+    const newsType = ctx.req.param('newsType') || 'all';
+    const response = await got(HOST);
+    const data = response.data;
+    const $ = load(data);
+    const newsList = $(newsTypeObj[newsType].selector).toArray();
+
+    const newsDetail = await Promise.all(
+        newsList.map((item) => {
+            const href = item.children[0].attribs.href;
+            return cache.tryGet(href, async () => {
+                const newsContent = await getContent(href);
+                return {
+                    title: item.children[0].children[0].data,
+                    description: newsContent.content,
+                    link: newsContent.newsPage,
+                    pubDate: timezone(parseDate(newsContent.newsTime, '时间：YYYY-MM-DD HH:mm:ss'), +8),
+                };
+            });
+        })
+    );
+    return {
+        title: `考试新闻发布(${newsTypeObj[newsType].name})`,
+        link: HOST,
+        description: `德阳人事考试网 考试新闻发布 (${newsTypeObj[newsType].name})`,
+        item: newsDetail,
+    };
+}
+
+export const route: Route = {
+    path: '/news/:newsType?',
+    categories: ['government'],
+    example: '/dykszx/news',
+    parameters: { newsType: '考试类型。默认新闻中心(all)' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['www.dykszx.com/'],
+            target: '/news/all',
+        },
+    ],
+    name: '考试新闻发布',
+    maintainers: ['zytomorrow'],
+    handler,
+    url: 'www.dykszx.com',
+    description: `| 新闻中心 | 公务员考试 | 事业单位 | （职）业资格、职称考试 | 其他 |
+| :------: | :------: | :------: |:------: |:------: |
+|   all   |   gwy   |  sydw | zyzc  | other |`,
+};

package/lib/routes/eagle/blog.ts

@@ -9,9 +9,20 @@ const cateList = new Set(['all', 'design-resources', 'learn-design', 'inside-eag
 
 export const route: Route = {
     path: '/blog/:cate?/:language?',
-    categories: ['design'],
+    categories: ['blog'],
     example: '/eagle/blog/en',
-    parameters: { cate: 'Category, get by URL, `all` by default', language: 'Language, `cn`, `tw`, `en`, `en` by default' },
+    parameters: {
+        cate: 'Category, get by URL, `all` by default',
+        language: {
+            description: 'Language',
+            options: [
+                { value: 'cn', label: 'cn' },
+                { value: 'tw', label: 'tw' },
+                { value: 'en', label: 'en' },
+            ],
+            default: 'en',
+        },
+    },
     features: {
         requireConfig: false,
         requirePuppeteer: false,

package/lib/routes/eastmoney/report/index.ts

@@ -1,8 +1,14 @@
 import { Route } from '@/types';
+import { getCurrentPath } from '@/utils/helpers';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
 import { parseDate } from '@/utils/parse-date';
+import { art } from '@/utils/render';
+import path from 'node:path';
+import { getRatingChangeStr, getEpsOrPeStr } from '../utils';
+
+const __dirname = getCurrentPath(import.meta.url);
 
 export const route: Route = {
     path: '/report/:category',
@@ -25,9 +31,9 @@ export const route: Route = {
     name: '研究报告',
     maintainers: ['syzq'],
     handler,
-    description: `| 策略报告       | 宏观研究    | 券商晨报     | 行业研究 |
-  | -------------- | ----------- | ------------ | -------- |
-  | strategyreport | macresearch | brokerreport | industry |`,
+    description: `| 策略报告       | 宏观研究    | 券商晨报     | 行业研究 | 个股研报 |
+  | -------------- | ----------- | ------------ | -------- | -------- |
+  | strategyreport | macresearch | brokerreport | industry | stock    |`,
 };
 
 async function handler(ctx) {
@@ -39,12 +45,14 @@ async function handler(ctx) {
         industry: '行业研报',
         macresearch: '宏观研究',
         strategyreport: '策略报告',
+        stock: '个股研报',
     };
     const linkType = {
         brokerreport: 'zw_brokerreport',
         industry: 'zw_industry',
         macresearch: 'zw_macresearch',
         strategyreport: 'zw_strategy',
+        stock: 'info',
     };
 
     const res = await got(`${baseUrl}/report/${category}`);
@@ -56,27 +64,67 @@ async function handler(ctx) {
             .match(/var initdata(.=?)(.*?);/)[2]
     );
 
-    const list = initData.data.map((item) => ({
-        title: `[${item.orgSName}]${item.title}`,
-        link: `${baseUrl}/report/${linkType[category]}.jshtml?encodeUrl=${item.encodeUrl}`,
-        pubDate: parseDate(item.publishDate),
-        author: item.researcher,
-    }));
+    const list = initData.data.map((item) => {
+        const stockName = category === 'stock' ? `[${item.stockName}]` : '';
+        return {
+            title: `[${item.orgSName}]${stockName}${item.title}`,
+            link: `${baseUrl}/report/${linkType[category]}` + (category === 'stock' ? `/${item.infoCode}.html` : `.jshtml?encodeUrl=${item.encodeUrl}`),
+            pubDate: parseDate(item.publishDate),
+            author: item.researcher,
+            originItem: item, // temp use
+        };
+    });
 
     const items = await Promise.all(
-        list.map((item) =>
-            cache.tryGet(item.link, async () => {
+        list.map((item) => {
+            const tempOriginItem = item.originItem;
+            delete item.originItem; // temp use
+
+            return cache.tryGet(item.link, async () => {
                 try {
                     const { data: response } = await got(item.link);
                     const $ = load(response);
-                    item.link = $('.pdf-link').attr('href');
-                    item.description = $('.ctx-content').text();
+
+                    if (category === 'stock') {
+                        const { title, stockName, stockCode, emRatingName, ratingChange, orgSName, indvInduName } = tempOriginItem;
+                        const ratingChangeStr = getRatingChangeStr(ratingChange);
+                        const currentYear = new Date().getFullYear();
+                        const nextYear = currentYear + 1;
+                        const description = $('.newsContent').html();
+                        const enclosureUrl = $('#ContentBody .rightlab').attr('href');
+                        const predictThisYearEps = getEpsOrPeStr(tempOriginItem.predictThisYearEps, 3);
+                        const predictThisYearPe = getEpsOrPeStr(tempOriginItem.predictThisYearPe, 2);
+                        const predictNextYearEps = getEpsOrPeStr(tempOriginItem.predictNextYearEps, 3);
+                        const predictNextYearPe = getEpsOrPeStr(tempOriginItem.predictNextYearPe, 2);
+
+                        item.enclosure_url = enclosureUrl;
+                        item.description = art(path.join(__dirname, '../templates/stock_description.art'), {
+                            title,
+                            stockName,
+                            stockCode,
+                            emRatingName,
+                            ratingChangeStr,
+                            description,
+                            orgSName,
+                            predictThisYearEps,
+                            predictThisYearPe,
+                            predictNextYearEps,
+                            predictNextYearPe,
+                            indvInduName,
+                            currentYear,
+                            nextYear,
+                            enclosureUrl,
+                        });
+                    } else {
+                        item.link = $('.pdf-link').attr('href');
+                        item.description = $('.ctx-content').text();
+                    }
                     return item;
                 } catch {
                     return item;
                 }
-            })
-        )
+            });
+        })
     );
 
     return {

package/lib/routes/eastmoney/templates/stock_description.art

@@ -0,0 +1,36 @@
+<table>
+  <tbody>
+    <tr>
+      <th rowspan="2">股票代码</th>
+      <th rowspan="2">股票简称</th>
+      <th rowspan="2">报告名称</th>
+      <th rowspan="2">东财评级</th>
+      <th rowspan="2">评级变动</th>
+      <th colspan="2">{{ currentYear }}盈利预测</th>
+      <th colspan="2">{{ nextYear }}盈利预测</th>
+      <th rowspan="2">行业</th>
+    </tr>
+    <tr>
+      <th>收益</th>
+      <th>市盈率</th>
+      <th>收益</th>
+      <th>市盈率</th>
+    </tr>
+    <tr>
+      <td>{{ stockCode }}</td>
+      <td>{{ stockName }}</td>
+      <td><a href="{{ enclosureUrl }}">{{ title }}</a></td>
+      <td>{{ emRatingName }}</td>
+      <td>{{ ratingChangeStr }}</td>
+      <td>{{ predictThisYearEps }}</td>
+      <td>{{ predictThisYearPe }}</td>
+      <td>{{ predictNextYearEps }}</td>
+      <td>{{ predictNextYearPe }}</td>
+      <td>{{ indvInduName }}</td>
+    </tr>
+  </tbody>
+</table>
+
+<div>
+    {{@ description }}
+</div>

package/lib/routes/eastmoney/utils.ts

@@ -0,0 +1,35 @@
+function getRatingChangeStr(ratingChange) {
+    let ratingChangeName = '';
+    switch (String(ratingChange)) {
+        case '0':
+            ratingChangeName = '调高';
+            break;
+        case '1':
+            ratingChangeName = '调低';
+            break;
+        case '2':
+            ratingChangeName = '首次';
+            break;
+        case '3':
+            ratingChangeName = '维持';
+            break;
+        case '4':
+            ratingChangeName = '无';
+            break;
+        default:
+            ratingChangeName = '-';
+            break;
+    }
+    return ratingChangeName;
+}
+
+// 按照东方财富的 js 规则：收益是保留三位小数，市盈率是保留两位小数
+function getEpsOrPeStr(epsOrPe, keepDecimal) {
+    let EpsOrPeStr = '';
+    if (epsOrPe !== undefined) {
+        EpsOrPeStr = epsOrPe === '' ? '' : Number(epsOrPe).toFixed(keepDecimal);
+    }
+    return EpsOrPeStr;
+}
+
+export { getRatingChangeStr, getEpsOrPeStr };

package/lib/routes/easynomad/joblist.ts

@@ -0,0 +1,56 @@
+import { Route, ViewType } from '@/types';
+import got from '@/utils/got';
+import MarkdownIt from 'markdown-it';
+const md = MarkdownIt({
+    html: true,
+    linkify: true,
+});
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    path: '/',
+    categories: ['other'],
+    view: ViewType.Notifications,
+    example: '/easynomad',
+    radar: [
+        {
+            source: ['easynomad.cn'],
+        },
+    ],
+    name: '远程工作列表',
+    maintainers: ['jiangsong216'],
+    handler,
+
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+};
+
+async function handler() {
+    const host = 'https://easynomad.cn';
+    const url = 'https://easynomad.cn/api/posts/list?limit=15&page=1&jobCategory=&contractType=';
+    const response = await got({
+        method: 'get',
+        url,
+    });
+    const data = response.data.data;
+
+    const items = data.map((item) => ({
+        title: item.jobTitle,
+        description: item.descContent ? md.render(item.descContent) : 'No description',
+        pubDate: parseDate(item.jobPublishTime),
+        link: item.url,
+    }));
+
+    return {
+        title: '轻松游牧-远程工作聚合列表',
+        description: '支持国内远程的招聘列表，远程全职，远程兼职',
+        link: host,
+        item: items,
+    };
+}

package/lib/routes/easynomad/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '轻松游牧-远程工作聚集地',
+    url: 'easynomad.cn',
+};

package/lib/routes/economist/espresso.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import got from '@/utils/got';
 import { load } from 'cheerio';
@@ -8,7 +8,8 @@ const link = 'https://www.economist.com/the-world-in-brief';
 
 export const route: Route = {
     path: '/espresso',
-    categories: ['traditional-media'],
+    categories: ['traditional-media', 'popular'],
+    view: ViewType.Articles,
     example: '/economist/espresso',
     parameters: {},
     features: {

package/lib/routes/economist/full.ts

@@ -1,4 +1,4 @@
-import { Route } from '@/types';
+import { Route, ViewType } from '@/types';
 import cache from '@/utils/cache';
 import parser from '@/utils/rss-parser';
 import got from '@/utils/got';
@@ -21,7 +21,8 @@ const getArticleDetail = (link) =>
 
 export const route: Route = {
     path: '/:endpoint',
-    categories: ['traditional-media'],
+    categories: ['traditional-media', 'popular'],
+    view: ViewType.Articles,
     example: '/economist/latest',
     parameters: { endpoint: 'Category name, can be found on the [official page](https://www.economist.com/rss). For example, https://www.economist.com/china/rss.xml to china' },
     features: {

package/lib/routes/epicgames/index.ts

@@ -12,7 +12,7 @@ export const route: Route = {
     path: '/freegames/:locale?/:country?',
     categories: ['game', 'popular'],
     view: ViewType.Notifications,
-    example: '/epicgames/freegames',
+    example: '/epicgames/freegames/en-US/US',
     parameters: {
         locale: {
             description: 'Locale',

package/lib/routes/espn/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'ESPN',
+    url: 'espn.com',
+};