rsshub 1.0.0-master.f6cb490 → 1.0.0-master.f6f0273

package/lib/api/index.ts

@@ -29,11 +29,6 @@ for (const path in docs.paths) {
     delete docs.paths[path];
 }
 app.get('/openapi.json', (ctx) => ctx.json(docs));
-app.get(
-    '/reference',
-    apiReference({
-        spec: { content: docs },
-    })
-);
+app.get('/reference', apiReference({ content: docs }));
 
 export default app;

package/lib/routes/2048/index.ts

@@ -67,31 +67,27 @@ async function handler(ctx) {
     const id = ctx.req.param('id') ?? '3';
 
     const rootUrl = 'https://hjd2048.com';
-
+    // 获取地址发布页指向的 URL
     const domainInfo = (await cache.tryGet('2048:domainInfo', async () => {
-        const response = await ofetch('https://hjd.tw', {
-            headers: {
-                accept: '*/*',
-            },
-        });
-        let $ = load(response);
-        const targetLink = new URL($('.address-list a').eq(0).attr('href'), 'https://hjd.tw').href;
-        const redirectResponse = await ofetch.raw(targetLink);
-        $ = load(redirectResponse._data);
-        return {
-            url: redirectResponse.url,
-            cookie:
-                $('script')
-                    .text()
-                    .match(/var safeid='(.*?)',/)?.[1] ?? '',
-        };
-    })) as { url: string; cookie: string };
-
-    const currentUrl = `${domainInfo.url}thread.php?fid-${id}.html`;
+        const response = await ofetch('https://2048.info');
+        const $ = load(response);
+        const onclickValue = $('.button').first().attr('onclick');
+        const targetUrl = onclickValue.match(/window\.open\('([^']+)'/)[1];
+
+        return { url: targetUrl };
+    })) as { url: string };
+    // 获取重定向后的url和safeid
+    const redirectResponse = await ofetch.raw(domainInfo.url);
+    const currentUrl = `${redirectResponse.url}thread.php?fid-${id}.html`;
+    const redirectPageContent = load(redirectResponse._data);
+    const safeId =
+        redirectPageContent('script')
+            .text()
+            .match(/var safeid='(.*?)',/)?.[1] ?? '';
 
     const response = await ofetch.raw(currentUrl, {
         headers: {
-            cookie: `_safe=${domainInfo.cookie}`,
+            cookie: `_safe=${safeId}`,
         },
     });
 
@@ -121,7 +117,7 @@ async function handler(ctx) {
             cache.tryGet(item.guid, async () => {
                 const detailResponse = await ofetch(item.link, {
                     headers: {
-                        cookie: `_safe=${domainInfo.cookie}`,
+                        cookie: `_safe=${safeId}`,
                     },
                 });
 
@@ -130,7 +126,12 @@ async function handler(ctx) {
                 content('.ads, .tips').remove();
 
                 content('ignore_js_op').each(function () {
-                    content(this).replaceWith(`<img src="${content(this).find('img').attr('src')}">`);
+                    const img = content(this).find('img');
+                    const originalSrc = img.attr('data-original');
+                    const fallbackSrc = img.attr('src');
+                    // 判断是否有 data-original 属性，若有则使用其值，否则使用 src 属性值
+                    const imgSrc = originalSrc || fallbackSrc;
+                    content(this).replaceWith(`<img src="${imgSrc}">`);
                 });
 
                 item.author = content('.fl.black').first().text();

package/lib/routes/anthropic/news.ts

@@ -1,4 +1,4 @@
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import cache from '@/utils/cache';
 import { Route } from '@/types';
@@ -10,28 +10,28 @@ export const route: Route = {
     parameters: {},
     radar: [
         {
-            source: ['anthropic.com'],
+            source: ['www.anthropic.com/news', 'www.anthropic.com'],
         },
     ],
     name: 'News',
     maintainers: ['etShaw-zh'],
     handler,
-    url: 'anthropic.com/news',
+    url: 'www.anthropic.com/news',
 };
 
 async function handler() {
-    const link = 'https://anthropic.com/news';
-    const response = await got(link);
-    const $ = load(response.body);
+    const link = 'https://www.anthropic.com/news';
+    const response = await ofetch(link);
+    const $ = load(response);
 
     const list = $('.contentFadeUp a')
         .toArray()
         .map((e) => {
             e = $(e);
-            const title = e.find('h3.PostCard_post-heading__KPsva').text().trim(); // Extract title
-            const href = e.attr('href'); // Extract link
-            const pubDate = e.find('.PostList_post-date__giqsu').text().trim(); // Extract publication date
-            const fullLink = href.startsWith('http') ? href : `https://anthropic.com${href}`; // Complete relative links
+            const title = e.find('h3[class^="PostCard_post-heading__"]').text().trim();
+            const href = e.attr('href');
+            const pubDate = e.find('div[class^="PostList_post-date__"]').text().trim();
+            const fullLink = href.startsWith('http') ? href : `https://www.anthropic.com${href}`;
             return {
                 title,
                 link: fullLink,
@@ -42,10 +42,24 @@ async function handler() {
     const out = await Promise.all(
         list.map((item) =>
             cache.tryGet(item.link, async () => {
-                const response = await got(item.link);
-                const $ = load(response.body);
+                const response = await ofetch(item.link);
+                const $ = load(response);
 
-                item.description = $('.text-b2.PostDetail_post-detail__uTcjp').html() || ''; // Full article content
+                $('div[class^="PostDetail_b-social-share"]').remove();
+
+                const content = $('div[class*="PostDetail_post-detail__"]');
+                content.find('img').each((_, e) => {
+                    const $e = $(e);
+                    $e.removeAttr('style srcset');
+                    const src = $e.attr('src');
+                    const params = new URLSearchParams(src);
+                    const newSrc = params.get('/_next/image?url');
+                    if (newSrc) {
+                        $e.attr('src', newSrc);
+                    }
+                });
+
+                item.description = content.html();
 
                 return item;
             })

package/lib/routes/asianfanfics/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Asianfanfics',
+    url: 'asianfanfics.com',
+    lang: 'en',
+};

package/lib/routes/asianfanfics/tag.ts

@@ -0,0 +1,89 @@
+import { DataItem, Route } from '@/types';
+
+import { config } from '@/config';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+
+// test url http://localhost:1200/asianfanfics/tag/milklove/N
+
+export const route: Route = {
+    path: '/tag/:tag/:type',
+    categories: ['reading'],
+    example: '/asianfanfics/tag/milklove/N',
+    parameters: {
+        tag: '标签',
+        type: '排序类型',
+    },
+    name: '标签',
+    maintainers: ['KazooTTT'],
+    radar: [
+        {
+            source: ['www.asianfanfics.com/browse/tag/:tag/:type'],
+            target: '/tag/:tag/:type',
+        },
+    ],
+    description: `匹配asianfanfics标签，支持排序类型：
+- L: Latest 最近更新
+- N: Newest 最近发布
+- O: Oldest 最早发布
+- C: Completed 已完成
+- OS: One Shots 短篇
+`,
+    handler,
+};
+
+type Type = 'L' | 'N' | 'O' | 'C' | 'OS';
+
+const typeToText = {
+    L: '最近更新',
+    N: '最近发布',
+    O: '最早发布',
+    C: '已完成',
+    OS: '短篇',
+};
+
+async function handler(ctx) {
+    const tag = ctx.req.param('tag');
+    const type = ctx.req.param('type') as Type;
+
+    if (!type || !['L', 'N', 'O', 'C', 'OS'].includes(type)) {
+        throw new Error('无效的排序类型');
+    }
+    const link = `https://www.asianfanfics.com/browse/tag/${tag}/${type}`;
+
+    const response = await ofetch(link, {
+        headers: {
+            'user-agent': config.trueUA,
+            Referer: 'https://www.asianfanfics.com/',
+        },
+    });
+    const $ = load(response);
+
+    const items: DataItem[] = $('.primary-container .excerpt')
+        .toArray()
+        .filter((element) => {
+            const $element = $(element);
+            return $element.find('.excerpt__title a').length > 0;
+        })
+        .map((element) => {
+            const $element = $(element);
+            const title = $element.find('.excerpt__title a').text();
+            const link = 'https://www.asianfanfics.com' + $element.find('.excerpt__title a').attr('href');
+            const author = $element.find('.excerpt__meta__name a').text().trim();
+            const pubDate = parseDate($element.find('time').attr('datetime') || '');
+
+            return {
+                title,
+                link,
+                author,
+                pubDate,
+            };
+        });
+
+    return {
+        title: `Asianfanfics - 标签：${tag} - ${typeToText[type]}`,
+        link,
+        item: items,
+    };
+}

package/lib/routes/asianfanfics/text-search.ts

@@ -0,0 +1,68 @@
+import { config } from '@/config';
+import { DataItem, Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+
+// test url http://localhost:1200/asianfanfics/text-search/milklove
+
+export const route: Route = {
+    path: '/text-search/:keyword',
+    categories: ['reading'],
+    example: '/asianfanfics/text-search/milklove',
+    parameters: {
+        keyword: '关键词',
+    },
+    name: '关键词',
+    maintainers: ['KazooTTT'],
+    radar: [
+        {
+            source: ['www.asianfanfics.com/browse/text_search?q=:keyword'],
+            target: '/text-search/:keyword',
+        },
+    ],
+    description: '匹配asianfanfics搜索关键词',
+    handler,
+};
+
+async function handler(ctx) {
+    const keyword = ctx.req.param('keyword');
+    if (keyword.trim() === '') {
+        throw new Error('关键词不能为空');
+    }
+    const link = `https://www.asianfanfics.com/browse/text_search?q=${keyword}+`;
+
+    const response = await ofetch(link, {
+        headers: {
+            'user-agent': config.trueUA,
+        },
+    });
+    const $ = load(response);
+
+    const items: DataItem[] = $('.primary-container .excerpt')
+        .toArray()
+        .filter((element) => {
+            const $element = $(element);
+            return $element.find('.excerpt__title a').length > 0;
+        })
+        .map((element) => {
+            const $element = $(element);
+            const title = $element.find('.excerpt__title a').text();
+            const link = 'https://www.asianfanfics.com' + $element.find('.excerpt__title a').attr('href');
+            const author = $element.find('.excerpt__meta__name a').text().trim();
+            const pubDate = parseDate($element.find('time').attr('datetime') || '');
+
+            return {
+                title,
+                link,
+                author,
+                pubDate,
+            };
+        });
+
+    return {
+        title: `Asianfanfics - 关键词：${keyword}`,
+        link,
+        item: items,
+    };
+}

package/lib/routes/blockworks/index.ts

@@ -0,0 +1,128 @@
+import { Route, Data, DataItem } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+import logger from '@/utils/logger';
+import parser from '@/utils/rss-parser';
+import { config } from '@/config';
+
+export const route: Route = {
+    path: '/',
+    categories: ['finance'],
+    example: '/blockworks',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['blockworks.co/'],
+            target: '/',
+        },
+    ],
+    name: 'News',
+    maintainers: ['pseudoyu'],
+    handler,
+    description: 'Blockworks news with full text support.',
+};
+
+async function handler(ctx): Promise<Data> {
+    const rssUrl = 'https://blockworks.co/feed';
+    const feed = await parser.parseURL(rssUrl);
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit'), 10) : 20;
+    // Limit to 20 items
+    const limitedItems = feed.items.slice(0, limit);
+
+    const buildId = await getBuildId();
+
+    const items = await Promise.all(
+        limitedItems
+            .map((item) => ({
+                ...item,
+                link: item.link?.split('?')[0],
+            }))
+            .map((item) =>
+                cache.tryGet(item.link!, async () => {
+                    // Get cached content or fetch new content
+                    const content = await extractFullText(item.link!.split('/').pop()!, buildId);
+
+                    return {
+                        title: item.title || 'Untitled',
+                        pubDate: item.isoDate ? parseDate(item.isoDate) : undefined,
+                        link: item.link,
+                        description: content.description || item.content || item.contentSnippet || item.summary || '',
+                        author: item.author,
+                        category: content.category,
+                        media: content.imageUrl
+                            ? {
+                                  content: { url: content.imageUrl },
+                              }
+                            : undefined,
+                    } as DataItem;
+                })
+            )
+    );
+
+    return {
+        title: feed.title || 'Blockworks News',
+        link: feed.link || 'https://blockworks.co',
+        description: feed.description || 'Latest news from Blockworks',
+        item: items,
+        language: feed.language || 'en',
+    };
+}
+
+async function extractFullText(slug: string, buildId: string): Promise<{ description: string; imageUrl: string; category: string[] }> {
+    try {
+        const response = await ofetch(`https://blockworks.co/_next/data/${buildId}/news/${slug}.json?slug=${slug}`);
+        const article = response.pageProps.article;
+        const $ = load(article.content, null, false);
+
+        // Remove promotional content at the end
+        $('hr').remove();
+        $('p > em, p > strong').each((_, el) => {
+            const $el = $(el);
+            if ($el.text().includes('To read full editions') || $el.text().includes('Get the news in your inbox')) {
+                $el.parent().remove();
+            }
+        });
+        $('ul.wp-block-list > li > a').each((_, el) => {
+            const $el = $(el);
+            if ($el.attr('href') === 'https://blockworks.co/newsletter/daily') {
+                $el.parent().parent().remove();
+            }
+        });
+
+        return {
+            description: $.html(),
+            imageUrl: article.imageUrl,
+            category: [...new Set([...article.categories, ...article.tags])],
+        };
+    } catch (error) {
+        logger.error('Error extracting full text from Blockworks:', error);
+        return { description: '', imageUrl: '', category: [] };
+    }
+}
+
+const getBuildId = () =>
+    cache.tryGet(
+        'blockworks:buildId',
+        async () => {
+            const response = await ofetch('https://blockworks.co');
+            const $ = load(response);
+
+            return (
+                $('script#__NEXT_DATA__')
+                    .text()
+                    ?.match(/"buildId":"(.*?)",/)?.[1] || ''
+            );
+        },
+        config.cache.routeExpire,
+        false
+    ) as Promise<string>;

package/lib/routes/blockworks/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Blockworks',
+    url: 'blockworks.co',
+    lang: 'en',
+};

package/lib/routes/cmu/andypavlo/blog.ts

@@ -0,0 +1,55 @@
+import { Route } from '@/types';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+async function getArticles() {
+    const url = 'https://www.cs.cmu.edu/~pavlo/blog/index.html';
+    const { data: res } = await got(url);
+    const $ = load(res);
+
+    const list = $('.row.mb-3')
+        .toArray()
+        .map((element) => {
+            const $item = $(element);
+            const $title = $item.find('h4 a');
+            const $date = $item.find('.text-muted');
+            const $description = $item.find('p');
+
+            return {
+                title: $title.text().trim(),
+                link: $title.attr('href'),
+                description: $description.text().trim(),
+                pubDate: parseDate($date.attr('title')),
+                guid: $title.attr('href'),
+            };
+        });
+    return list;
+}
+
+export const route: Route = {
+    path: '/andypavlo/blog',
+    categories: ['blog'],
+    example: '/cmu/andypavlo/blog',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Andy Pavlo Blog',
+    maintainers: ['mocusez'],
+    handler,
+};
+
+async function handler() {
+    const articles = await getArticles();
+    return {
+        title: 'Andy Pavlo - Carnegie Mellon University',
+        link: 'https://www.cs.cmu.edu/~pavlo/blog/index.html',
+        item: articles,
+    };
+}

package/lib/routes/cmu/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Carnegie Mellon University',
+    url: 'www.cmu.edu',
+    lang: 'en',
+};

package/lib/routes/coindesk/{index.ts → consensus-magazine.ts}

@@ -1,6 +1,8 @@
 import { Route } from '@/types';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
+import cache from '@/utils/cache';
 import { load } from 'cheerio';
+import { parseItem } from './utils';
 const rootUrl = 'https://www.coindesk.com';
 
 export const route: Route = {
@@ -27,29 +29,23 @@ export const route: Route = {
     url: 'coindesk.com/',
 };
 
-async function handler(ctx) {
-    const channel = ctx.req.param('channel') ?? 'consensus-magazine';
+async function handler() {
+    const channel = 'consensus-magazine';
 
-    const response = await got.get(`${rootUrl}/${channel}/`);
-    const $ = load(response.data);
-    const content = JSON.parse(
-        $('#fusion-metadata')
-            .text()
-            .match(/Fusion\.contentCache=(.*?);Fusion\.layout/)[1]
-    );
+    const response = await ofetch(`${rootUrl}/${channel}`);
+    const $ = load(response);
 
-    const o1 = content['websked-collections'];
-    // Object key names are different every week
-    const articles = o1[Object.keys(o1)[2]];
+    const list = $('div h2')
+        .toArray()
+        .map((item) => {
+            const $item = $(item);
+            return {
+                title: $item.text(),
+                link: rootUrl + $item.parent().attr('href'),
+            };
+        });
 
-    const list = articles.data;
-
-    const items = list.map((item) => ({
-        title: item.headlines.basic,
-        link: rootUrl + item.canonical_url,
-        description: item.subheadlines.basic,
-        pubDate: item.display_date,
-    }));
+    const items = await Promise.all(list.map((item) => cache.tryGet(item.link, () => parseItem(item))));
 
     return {
         title: 'CoinDesk Consensus Magazine',

package/lib/routes/coindesk/namespace.ts

@@ -1,7 +1,8 @@
 import type { Namespace } from '@/types';
 
 export const namespace: Namespace = {
-    name: 'CoinDesk Consensus Magazine',
+    name: 'CoinDesk',
     url: 'coindesk.com',
     lang: 'en',
+    description: 'CoinDesk is a news site specializing in bitcoin and digital currencies, delivering news, analysis, and information about the blockchain ecosystem.',
 };

package/lib/routes/coindesk/news.ts

@@ -0,0 +1,47 @@
+import { Route, Data, DataItem } from '@/types';
+import cache from '@/utils/cache';
+import parser from '@/utils/rss-parser';
+import { parseItem } from './utils';
+
+export const route: Route = {
+    path: '/news',
+    categories: ['finance'],
+    example: '/coindesk/news',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'News',
+    maintainers: ['pseudoyu'],
+    handler,
+    radar: [
+        {
+            source: ['coindesk.com/'],
+            target: '/news',
+        },
+    ],
+    description: 'Get latest news from CoinDesk with full text.',
+};
+
+async function handler(): Promise<Data> {
+    const rssUrl = 'https://feeds.feedburner.com/Coindesk';
+    const feed = await parser.parseURL(rssUrl);
+
+    const items = await Promise.all(feed.items.map((item) => cache.tryGet(item.link, () => parseItem(item))));
+
+    // Filter out null items
+    const validItems = items.filter((item): item is DataItem => item !== null);
+
+    return {
+        title: feed.title || 'CoinDesk News',
+        link: feed.link || 'https://coindesk.com',
+        description: feed.description || 'Latest news from CoinDesk',
+        language: feed.language || 'en',
+        item: validItems,
+    };
+}

package/lib/routes/coindesk/utils.ts

@@ -0,0 +1,26 @@
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+export const parseItem = async (item) => {
+    const response = await ofetch(item.link);
+    const $ = load(response);
+    const ldJson = JSON.parse($('script[type="application/ld+json"]').text());
+
+    $('.article-ad, #strategy-rules-player-wrapper, [data-module-name="newsletter-article-sign-up-module"], div.flex.flex-col.gap-2').remove();
+    const cover = $('.article-content-wrapper figure');
+    cover.find('img').attr('src', cover.find('img').attr('url')?.split('?')[0]);
+    cover.find('img').removeAttr('style srcset url');
+
+    item.description =
+        cover.parent().html() +
+        $('.document-body')
+            .toArray()
+            .map((item) => $(item).html())
+            .join('');
+    item.pubDate = parseDate(ldJson.datePublished);
+    item.author = ldJson.author.map((a) => ({ name: a.name }));
+    item.image = ldJson.image.url.split('?')[0];
+
+    return item;
+};