rsshub 1.0.0-master.f9003f9 → 1.0.0-master.f917bdc

package/lib/routes/cffex/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '中国金融期货交易所',
+    url: 'cffex.com.cn',
+};

package/lib/routes/cfr/index.ts

@@ -0,0 +1,58 @@
+import type { Data, Route } from '@/types';
+import type { Context } from 'hono';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { asyncPoolAll, getDataItem } from './utils';
+
+export const route: Route = {
+    path: '/:category/:subCategory?',
+    categories: ['traditional-media'],
+    parameters: {
+        category: 'category, find it in the URL',
+        subCategory: 'sub-category, find it in the URL',
+    },
+    example: '/cfr/asia',
+    name: 'News',
+    maintainers: ['KarasuShin'],
+    handler,
+    radar: [
+        {
+            source: ['www.cfr.org/:category', 'www.cfr.org/:category/:subCategory'],
+            target: '/:category/:subCategory?',
+        },
+    ],
+    features: {
+        antiCrawler: true,
+    },
+};
+
+async function handler(ctx: Context): Promise<Data> {
+    const { category, subCategory } = ctx.req.param();
+
+    const origin = 'https://www.cfr.org';
+    let link = `${origin}/${category}`;
+    if (subCategory) {
+        link += `/${subCategory}`;
+    }
+    const res = await ofetch(link);
+
+    const $ = load(res);
+
+    const selectorMap: {
+        [key: string]: string;
+    } = {
+        podcasts: '.episode-content__title a',
+        blog: '.card-series__content-link',
+        'books-reports': '.card-article__link',
+    };
+
+    const listSelector = selectorMap[category] ?? '.card-article-large__link';
+
+    const items = await asyncPoolAll(5, $(listSelector).toArray(), async (item) => await getDataItem($(item).attr('href')!));
+
+    return {
+        title: $('head title').text().replace(' | Council on Foreign Relations', ''),
+        link,
+        item: items,
+    };
+}

package/lib/routes/cfr/namespace.ts

@@ -0,0 +1,6 @@
+import { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Council on Foreign Relations',
+    url: 'www.cfr.org',
+};

package/lib/routes/cfr/types.ts

@@ -0,0 +1,30 @@
+export interface LinkData {
+    '@context': string;
+    '@graph': {
+        '@type': string;
+        headline: string;
+        name: string;
+        about: string;
+        description: string;
+        image: {
+            '@type': string;
+            representativeOfPage: string;
+            url: string;
+        };
+        datePublished: string;
+        dateModified: string;
+        author: {
+            '@type': string;
+            name: string;
+            url: string;
+        };
+    }[];
+}
+
+export interface VideoSetup {
+    techOrder: string[];
+    sources: {
+        type: string;
+        src: string;
+    }[];
+}

package/lib/routes/cfr/utils.ts

@@ -0,0 +1,284 @@
+import { type Cheerio, type CheerioAPI, type Element, load } from 'cheerio';
+import ofetch from '@/utils/ofetch';
+import type { DataItem } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import cache from '@/utils/cache';
+import type { LinkData, VideoSetup } from './types';
+import asyncPool from 'tiny-async-pool';
+
+export function getDataItem(href: string) {
+    const origin = 'https://www.cfr.org';
+    const link = `${origin}${href}`;
+
+    return cache.tryGet(link, async () => {
+        const prefix = href?.split('/')[1];
+        const res = await ofetch(link);
+        const $ = load(res);
+
+        let dataItem: DataItem;
+
+        switch (prefix) {
+            case 'article':
+                dataItem = parseArticle($);
+                break;
+            case 'blog':
+                dataItem = parseBlog($);
+                break;
+            case 'book':
+                dataItem = parseBook($);
+                break;
+            case 'conference-calls':
+                dataItem = parseConferenceCalls($);
+                break;
+            case 'event':
+                dataItem = parseEvent($);
+                break;
+            case 'backgrounder':
+                dataItem = parseBackgrounder($);
+                break;
+            case 'podcasts':
+                dataItem = parsePodcasts($);
+                break;
+            case 'task-force-report':
+                dataItem = parseTaskForceReport($);
+                break;
+            case 'timeline':
+                dataItem = parseTimeline($);
+                break;
+            case 'video':
+                dataItem = parseVideo($);
+                break;
+            default:
+                dataItem = parseDefault($);
+        }
+
+        return {
+            ...dataItem,
+            link,
+        };
+    }) as Promise<DataItem>;
+}
+
+function parseArticle($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+    let description = parseDescription($('.body-content'), $);
+    const $articleHeader = $('.article-header__image');
+    if ($articleHeader.length) {
+        description = `<figure>${$articleHeader.html()}</figure><br>${description}`;
+    }
+    return {
+        title: linkData?.title ?? $('.article-header__title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parseBlog($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+    let description = parseDescription($('.body-content'), $);
+    const figure = $('.article-header-blog__figure');
+    if (figure.length) {
+        description = `<figure>${figure.html()}</figure><br>${description}`;
+    }
+    return {
+        title: linkData?.title ?? $('.article-header-blog__title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parseBook($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+    let description = parseDescription($('.body-content'), $);
+    const sectionTop = $('.article-header__section-top');
+    description = `${sectionTop.html()}<br>${description}`;
+
+    return {
+        title: linkData?.title ?? $('.article-header__title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parseConferenceCalls($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+    const description = parseDescription($('.podcast-body').last(), $);
+    return {
+        title: linkData?.title ?? $('head title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parseEvent($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+    let description = parseDescription($('.body-content'), $);
+    const videoIfame = getVideoIframe($('.msp-event-video'));
+    if (videoIfame) {
+        description = `${videoIfame}<br>${description}`;
+    }
+
+    return {
+        title: linkData?.title ?? $('.msp-event-header-past__title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parseBackgrounder($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+    let description = parseDescription($('.main-wrapper__article-body .body-content'), $);
+    const summary = $('.main-wrapper__article-body .summary').html();
+    if (summary) {
+        description = `${summary}<br>${description}`;
+    }
+    const figure = $('.article-header-backgrounder__figure');
+    if (figure.length) {
+        description = `<figure>${figure.html()}</figure><br>${description}`;
+    }
+
+    return {
+        title: linkData?.title ?? $('.article-header-backgrounder__title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parsePodcasts($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+    let description = $('.body-content').first().html() ?? '';
+    const audioSrc = $('#player-default').attr('src');
+    if (audioSrc) {
+        description = `<audio controls src="${audioSrc}"></audio><br>${description}`;
+    }
+    return {
+        title: linkData?.title ?? $('head title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+        enclosure_url: audioSrc,
+        enclosure_type: 'audio/mpeg',
+    };
+}
+
+function parseTaskForceReport($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+
+    let description = '';
+
+    $('.main-content').each((_, ele) => {
+        const $ele = $(ele);
+        const content = $ele.find('.content_area').html() ?? '';
+        description += `${content}<br>`;
+    });
+
+    return {
+        title: linkData?.title ?? $('.hero__title').remove('.subtitle').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parseTimeline($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+
+    const $description = $('.timeline-slides');
+    $description.find('.timeline-slide__shadow').remove();
+    $description.find('.field--image').each((_, ele) => {
+        $(ele).replaceWith($(ele).find('img'));
+    });
+    let description = $description.find('.timeline-intro__description').html() ?? '';
+    for (const item of $description.find('.timeline-slide__content').toArray()) {
+        const $item = $(item);
+        $item.find('.timeline-slide__dates-header').replaceWith('<h1>' + $item.find('.timeline-slide__dates-header').text() + '</h1>');
+        $item.find('.timeline-slide__dates').replaceWith('<h2>' + $item.find('.timeline-slide__dates').text() + '</h2>');
+        description += `<br>${$item.html()}`;
+    }
+    return {
+        title: linkData?.title ?? $('.timeline-header__title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parseVideo($: CheerioAPI): DataItem {
+    const linkData = parseLinkData($);
+    let description = parseDescription($('.body-content'), $);
+    const $articleHeader = $('.article-header__image');
+    const videoIfame = getVideoIframe($articleHeader);
+    if (videoIfame) {
+        description = `${videoIfame}<br>${description}`;
+    }
+
+    return {
+        title: linkData?.title ?? $('.article-header__title').text(),
+        pubDate: linkData?.pubDate,
+        description,
+    };
+}
+
+function parseDefault($): DataItem {
+    if ($('.body-content').length) {
+        return parseArticle($);
+    }
+    const linkData = parseLinkData($);
+    return {
+        title: linkData?.title ?? $('head title').text(),
+        pubDate: linkData?.pubDate,
+    };
+}
+
+function parseLinkData($: CheerioAPI) {
+    try {
+        const data = (<LinkData>JSON.parse($('script[type="application/ld+json"]').text()))['@graph'][0];
+
+        return {
+            title: data.name,
+            pubDate: parseDate(data.dateModified),
+        };
+    } catch {
+        // ignore
+    }
+}
+
+function getVideoIframe($ele: Cheerio<Element>) {
+    const setup = $ele.find('video').data('setup') as VideoSetup;
+    if (setup) {
+        const youtubeSource = setup.sources.find((i) => i.type === 'video/youtube');
+        if (youtubeSource) {
+            const videoId = youtubeSource.src.match(/\?v=([^&]+)/)?.[1];
+            if (videoId) {
+                return `<iframe src="https://www.youtube-nocookie.com/embed/${videoId}" width="640" height="360" frameborder="0" allowfullscreen></iframe>`;
+            }
+        }
+    }
+}
+
+function parseDescription($description: Cheerio<Element>, $: CheerioAPI) {
+    $description.find('.desktop-only').remove();
+    $description.find('.mobile-only').remove();
+    $description.find('.newsletter-tout').remove();
+    $description.find('.carousel-gallery').remove();
+    $description.find('svg').remove();
+    $description.find('.field--image').each((_, ele) => {
+        $(ele).replaceWith($(ele).find('img'));
+    });
+    $description.find('.video-embed').each((_, ele) => {
+        const $ele = $(ele);
+        const videoIframe = getVideoIframe($ele);
+        if (videoIframe) {
+            $ele.replaceWith(videoIframe);
+        }
+    });
+
+    const description = $description.html() ?? '';
+
+    return description;
+}
+
+export async function asyncPoolAll<IN, OUT>(poolLimit: number, array: readonly IN[], iteratorFn: (generator: IN) => Promise<OUT>) {
+    const results: Awaited<OUT[]> = [];
+    for await (const result of asyncPool(poolLimit, array, iteratorFn)) {
+        results.push(result);
+    }
+    return results;
+}

package/lib/routes/chaping/newsflash.ts

@@ -1,5 +1,5 @@
 import { Route } from '@/types';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { parseDate } from '@/utils/parse-date';
 
 const host = 'https://chaping.cn';
@@ -30,7 +30,7 @@ export const route: Route = {
 
 async function handler() {
     const newflashAPI = `${host}/api/official/information/newsflash?page=1&limit=21`;
-    const response = await got(newflashAPI).json();
+    const response = await ofetch(newflashAPI);
     const data = response.data;
 
     return {

package/lib/routes/chlinlearn/daily-blog.ts

@@ -0,0 +1,48 @@
+import { Route } from '@/types';
+import ofetch from '@/utils/ofetch'; // 统一使用的请求库
+import { parseDate } from '@/utils/parse-date'; // 解析日期的工具函数
+import timezone from '@/utils/timezone';
+
+export const route: Route = {
+    path: '/daily-blog',
+    name: '值得一读技术博客',
+    maintainers: ['huyyi'],
+    categories: ['programming'],
+    example: '/chlinlearn/daily-blog',
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['daily-blog.chlinlearn.top/blogs/*'],
+            target: '/chlinlearn/daily-blog',
+        },
+    ],
+    handler: async () => {
+        const data = await ofetch('https://daily-blog.chlinlearn.top/api/daily-blog/getBlogs/new?type=new&pageNum=1&pageSize=20', {
+            headers: {
+                Referer: 'https://daily-blog.chlinlearn.top/blogs/1',
+            },
+        });
+        const items = data.rows.map((item) => ({
+            title: item.title,
+            link: item.url,
+            author: item.author,
+            img: item.icon,
+            pubDate: timezone(parseDate(item.publishTime), +8),
+        }));
+        return {
+            // 源标题
+            title: '值得一读技术博客',
+            // 源链接
+            link: 'https://daily-blog.chlinlearn.top/blogs/1',
+            // 源文章
+            item: items,
+        };
+    },
+};

package/lib/routes/chlinlearn/namespcae.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'chlinlearn的技术博客',
+    url: 'daily-blog.chlinlearn.top',
+};

package/lib/routes/cnbc/rss.ts

@@ -65,7 +65,7 @@ async function handler(ctx) {
                     }
 
                     const meta = JSON.parse($('[type=application/ld+json]').last().text());
-                    item.author = meta.author ? meta.author.name ?? meta.author.map((a) => a.name).join(', ') : null;
+                    item.author = meta.author ? (meta.author.name ?? meta.author.map((a) => a.name).join(', ')) : null;
                     item.category = meta.keywords;
 
                     return item;

package/lib/routes/codeforces/contests.ts

@@ -2,7 +2,7 @@ import { Route } from '@/types';
 import { getCurrentPath } from '@/utils/helpers';
 const __dirname = getCurrentPath(import.meta.url);
 
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import path from 'node:path';
 import { art } from '@/utils/render';
 
@@ -46,7 +46,7 @@ export const route: Route = {
 };
 
 async function handler() {
-    const contestsData = await got.get(contestAPI).json();
+    const contestsData = await ofetch(contestAPI);
     const contests = contestsData.result;
 
     const items = contests

package/lib/routes/codeforces/recent-actions.ts

@@ -1,5 +1,5 @@
 import { Route } from '@/types';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 
 export const route: Route = {
@@ -30,7 +30,7 @@ export const route: Route = {
 async function handler(ctx) {
     const minRating = ctx.req.param('minrating') || 1;
 
-    const rsp = await got.get('https://codeforces.com/api/recentActions?maxCount=100').json();
+    const rsp = await ofetch('https://codeforces.com/api/recentActions?maxCount=100');
 
     const actions = rsp.result.map((action) => {
         const pubDate = new Date(action.timeSeconds * 1000);

package/lib/routes/copymanga/comic.ts

@@ -34,7 +34,7 @@ async function handler(ctx) {
     // 用于控制返回的章节数量
     const chapterCnt = Number(ctx.req.param('chapterCnt') || 10);
     // 直接调用拷贝漫画的接口
-    const host = 'copymanga.site';
+    const host = 'copymanga.tv';
     const baseUrl = `https://${host}`;
     const apiBaseUrl = `https://api.${host}`;
     const strBaseUrl = `${apiBaseUrl}/api/v3/comic/${id}/group/default/chapters`;
@@ -76,6 +76,7 @@ async function handler(ctx) {
             chapters = chapters
                 .map(({ comic_path_word, uuid, name, size, datetime_created, ordered /* , index*/ }) => ({
                     link: `${baseUrl}/comic/${comic_path_word}/chapter/${uuid}`,
+                    guid: `https://copymanga.site/comic/${comic_path_word}/chapter/${uuid}`,
                     uuid,
                     title: name,
                     size,
@@ -117,6 +118,7 @@ async function handler(ctx) {
 
         return {
             link: chapter.link,
+            guid: chapter.guid,
             title: chapter.title,
             description: art(path.join(__dirname, './templates/comic.art'), {
                 size: chapter.size,

package/lib/routes/dangdang/namespace.ts

@@ -0,0 +1,6 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '当当开放平台',
+    url: 'open.dangdang.com',
+};

package/lib/routes/dangdang/notice.ts

@@ -0,0 +1,69 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+
+const typeMap = {
+    0: '全部',
+    1: '其他',
+    2: '规则变更',
+};
+
+/**
+ *
+ * @param ctx {import('koa').Context}
+ */
+export const route: Route = {
+    path: '/notice/:type?',
+    categories: ['programming'],
+    example: '/dangdang/notice/1',
+    parameters: { type: '公告分类，默认为全部' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '公告',
+    maintainers: ['353325487'],
+    handler,
+    description: `| 类型     | type |
+  | -------- | ---- |
+  | 全部      | 0    |
+  | 其他      | 1    |
+  | 规则变更   | 2    |`,
+};
+
+async function handler(ctx) {
+    const type = ctx.req.param('type');
+    const url = `https://open.dangdang.com/op-api/developer-platform/document/menu/list?categoryId=3&type=${type > 0 ? typeMap[type] : ''}`;
+    const response = await got({ method: 'get', url });
+
+    const list = response.data.data.documentMenu.map((item) => ({
+        title: item.title,
+        description: item.type,
+        documentId: item.documentId,
+        source: `https://open.dangdang.com/op-api/developer-platform/document/info/get?document_id=${item.documentId}`,
+        link: `https://open.dangdang.com/home/notice/message/1/${item.documentId}`,
+        pubDate: timezone(parseDate(item.modifyTime), +8),
+    }));
+
+    const result = await Promise.all(
+        list.map((item) =>
+            cache.tryGet(item.source, async () => {
+                const itemResponse = await got(item.source);
+                item.description = itemResponse.data.data.documentContentList[0].content;
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: `当当开放平台 - ${typeMap[type] || typeMap[0]}`,
+        link: `https://open.dangdang.com/home/notice/message/1`,
+        item: result,
+    };
+}

package/lib/routes/dataguidance/index.ts

@@ -0,0 +1,66 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+export const route: Route = {
+    name: 'News',
+    example: '/dataguidance/news',
+    path: '/news',
+    radar: [
+        {
+            source: ['dataguidance.com/search/news'],
+        },
+    ],
+    maintainers: ['harveyqiu'],
+    handler,
+    url: 'dataguidance.com/news',
+};
+
+async function handler() {
+    const rootUrl = 'https://www.dataguidance.com';
+    const currentUrl = `${rootUrl}/search/news/`;
+
+    const response = await got({
+        method: 'get',
+        url: currentUrl,
+    });
+
+    const $ = load(response.data);
+
+    let items = $('.field-name-title')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            const a = item.find('a').first();
+
+            return {
+                title: a.text(),
+                link: `${rootUrl}${a.attr('href')}`,
+            };
+        });
+
+    items = await Promise.all(
+        items.map((item) =>
+            cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: item.link,
+                });
+
+                const content = load(detailResponse.data);
+                item.pubDate = parseDate(content('.field-name-post-date').text());
+                item.description = content('.field-name-body').html();
+
+                return item;
+            })
+        )
+    );
+
+    return {
+        title: 'Data Guidance News',
+        link: currentUrl,
+        item: items,
+    };
+}

package/lib/routes/dataguidance/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'DataGuidance',
+    url: 'dataguidance.com',
+    categories: ['other'],
+};