rsshub 1.0.0-master.f6cb490 → 1.0.0-master.f6f0273

package/lib/routes/cointelegraph/index.ts

@@ -0,0 +1,106 @@
+import { Route, Data, DataItem } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+import logger from '@/utils/logger';
+import parser from '@/utils/rss-parser';
+
+export const route: Route = {
+    path: '/',
+    categories: ['finance'],
+    example: '/cointelegraph',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'News',
+    maintainers: ['pseudoyu'],
+    handler,
+    radar: [
+        {
+            source: ['cointelegraph.com/'],
+            target: '/',
+        },
+    ],
+    description: 'Get latest news from Cointelegraph with full text.',
+};
+
+async function handler(): Promise<Data> {
+    const rssUrl = 'https://cointelegraph.com/rss';
+    const feed = await parser.parseURL(rssUrl);
+
+    const items = await Promise.all(
+        feed.items
+            .filter((item) => item.link && /\/news|\/explained|\/innovation-circle/.test(item.link))
+            .map((item) => ({
+                ...item,
+                link: item.link?.split('?')[0],
+            }))
+            .map((item) =>
+                cache.tryGet(item.link!, async () => {
+                    const link = item.link!;
+
+                    // Extract full text
+                    const fullText = await extractFullText(link);
+
+                    if (!fullText) {
+                        logger.warn(`Failed to extract content from ${link}`);
+                    }
+
+                    // Create article item
+                    return {
+                        title: item.title || 'Untitled',
+                        description: fullText || item.content,
+                        pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
+                        link,
+                        author: item.creator || 'CoinTelegraph',
+                        category: item.categories?.map((c) => c.trim()) || [],
+                        image: item.enclosure?.url,
+                    } as DataItem;
+                })
+            )
+    );
+
+    // Filter out null items
+    const validItems = items.filter((item): item is DataItem => item !== null);
+
+    return {
+        title: feed.title || 'CoinTelegraph News',
+        link: feed.link || 'https://cointelegraph.com',
+        description: feed.description || 'Latest news from CoinTelegraph',
+        language: feed.language || 'en',
+        item: validItems,
+    };
+}
+
+async function extractFullText(url: string): Promise<string | null> {
+    try {
+        const response = await ofetch(url);
+        const $ = load(response);
+        const nuxtData = $('script:contains("window.__NUXT__")').text();
+        const fullText = JSON.parse(nuxtData.match(/\.fullText=(".*?");/)?.[1] || '{}');
+        const cover = $('.post-cover__image');
+
+        // Remove unwanted elements
+        cover.find('source').remove();
+        cover.find('img').removeAttr('srcset');
+        cover.find('img').attr(
+            'src',
+            cover
+                .find('img')
+                .attr('src')
+                ?.match(/(https:\/\/s3\.cointelegraph\.com\/.+)/)?.[1] || ''
+        );
+
+        return cover.html() + fullText || null;
+    } catch (error) {
+        logger.error(`Error fetching article content: ${error}`);
+        return null;
+    }
+}

package/lib/routes/cointelegraph/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Cointelegraph',
+    url: 'cointelegraph.com',
+    lang: 'en',
+};

package/lib/routes/collabo-cafe/category.ts

@@ -0,0 +1,37 @@
+import { Data, Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { Context } from 'hono';
+import { parseItems } from './parser';
+
+export const handler = async (ctx: Context): Promise<Data | null> => {
+    const { category } = ctx.req.param();
+    const baseUrl = `https://collabo-cafe.com/events/category/${category}`;
+    const res = await ofetch(baseUrl);
+    const $ = load(res);
+    const items = parseItems($);
+
+    return {
+        title: '分类',
+        link: baseUrl,
+        item: items,
+    };
+};
+
+export const route: Route = {
+    path: '/category/:category',
+    categories: ['anime'],
+    example: '/collabo-cafe/category/cafe',
+    parameters: { category: 'Category, refer to the original website (ジャンル別)' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '分类',
+    maintainers: ['cokemine'],
+    handler,
+};

package/lib/routes/collabo-cafe/index.ts

@@ -0,0 +1,35 @@
+import { Data, Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { parseItems } from './parser';
+
+export const handler = async (): Promise<Data | null> => {
+    const baseUrl = 'https://collabo-cafe.com/';
+    const res = await ofetch(baseUrl);
+    const $ = load(res);
+    const items = parseItems($);
+
+    return {
+        title: '全部文章',
+        link: baseUrl,
+        item: items,
+    };
+};
+
+export const route: Route = {
+    path: '/',
+    categories: ['anime'],
+    example: '/collabo-cafe/',
+    parameters: undefined,
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '全部文章',
+    maintainers: ['cokemine'],
+    handler,
+};

package/lib/routes/collabo-cafe/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'コラボカフェ',
+    url: 'collabo-cafe.com',
+    description: 'コラボカフェ - アニメ・漫画・ゲームのコラボ情報一覧まとめ',
+    lang: 'ja',
+    categories: ['anime'],
+};

package/lib/routes/collabo-cafe/parser.ts

@@ -0,0 +1,29 @@
+import { type DataItem } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import { CheerioAPI } from 'cheerio';
+
+export function parseItems($: CheerioAPI): DataItem[] {
+    return $('div.top-post-list article')
+        .toArray()
+        .map((el) => {
+            const $el = $(el);
+            const a = $el.find('a').first();
+            const title = a.attr('title')!;
+            const link = a.attr('href');
+            const pubDate = parseDate($el.find('span.date.gf.updated').text());
+            const author = $el.find('span.author span.fn').text();
+            const category = [$el.find('span.cat-name').text()];
+            const description = $el.find('div.description p').text();
+            const image = $el.find('img').attr('data-src');
+            return {
+                title,
+                link,
+                pubDate,
+                author,
+                category,
+                description,
+                image,
+                banner: image,
+            };
+        });
+}

package/lib/routes/collabo-cafe/tag.ts

@@ -0,0 +1,37 @@
+import { Data, Route } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { load } from 'cheerio';
+import { Context } from 'hono';
+import { parseItems } from './parser';
+
+export const handler = async (ctx: Context): Promise<Data | null> => {
+    const { tag } = ctx.req.param();
+    const baseUrl = `https://collabo-cafe.com/events/tag/${tag}`;
+    const res = await ofetch(baseUrl);
+    const $ = load(res);
+    const items = parseItems($);
+
+    return {
+        title: '标签',
+        link: baseUrl,
+        item: items,
+    };
+};
+
+export const route: Route = {
+    path: '/tag/:tag',
+    categories: ['anime'],
+    example: '/collabo-cafe/tag/ikebukuro',
+    parameters: { tag: 'Tag, refer to the original website (開催地域別)' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '标签',
+    maintainers: ['cokemine'],
+    handler,
+};

package/lib/routes/cryptoslate/index.ts

@@ -0,0 +1,98 @@
+import { Route, Data } from '@/types';
+import { parseDate } from '@/utils/parse-date';
+import logger from '@/utils/logger';
+import parser from '@/utils/rss-parser';
+import { load } from 'cheerio';
+
+export const route: Route = {
+    path: '/',
+    categories: ['finance'],
+    example: '/cryptoslate',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'News',
+    maintainers: ['pseudoyu'],
+    handler,
+    radar: [
+        {
+            source: ['cryptoslate.com/'],
+            target: '/',
+        },
+    ],
+    description: 'Get latest news from CryptoSlate.',
+};
+
+async function handler(ctx): Promise<Data> {
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 20;
+    const rssUrl = 'https://cryptoslate.com/feed/';
+
+    const feed = await parser.parseURL(rssUrl);
+
+    const items = feed.items
+        .filter((item) => !item.link?.includes('/feed') && !item.link?.includes('#respond'))
+        .slice(0, limit)
+        .map((item) => {
+            if (!item.link) {
+                return {};
+            }
+
+            try {
+                // Clean URL by removing query parameters
+                const cleanUrl = item.link.split('?')[0];
+
+                return {
+                    title: item.title || 'Untitled',
+                    link: cleanUrl,
+                    pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
+                    description: extractFullTextFromRSS(item),
+                    author: item.creator || 'CryptoSlate',
+                    category: item.categories || [],
+                    guid: item.guid || item.link,
+                    image: item.enclosure?.url,
+                };
+            } catch (error: any) {
+                logger.warn(`Couldn't process article from CryptoSlate: ${item.link}: ${error.message}`);
+                return {};
+            }
+        });
+
+    // Filter out empty items
+    const filteredItems = items.filter((item) => item && Object.keys(item).length > 0);
+
+    return {
+        title: feed.title || 'CryptoSlate',
+        link: feed.link || 'https://cryptoslate.com',
+        description: feed.description || 'Latest news from CryptoSlate',
+        item: filteredItems,
+        language: feed.language || 'en',
+        image: feed.image?.url,
+    } as Data;
+}
+
+function extractFullTextFromRSS(entry: any): string | null {
+    try {
+        const contentEncoded = entry['content:encoded'] || entry['content:encodedSnippet'] || entry.content || entry.contentSnippet;
+
+        if (!contentEncoded) {
+            return null;
+        }
+
+        const $ = load(contentEncoded);
+
+        // Remove unwanted elements
+        $('img').remove();
+        $('figure').remove();
+
+        return $.html() || null;
+    } catch (error) {
+        logger.error(`Error extracting full text from RSS: ${error}`);
+        return null;
+    }
+}

package/lib/routes/cryptoslate/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'CryptoSlate',
+    url: 'cryptoslate.com',
+    lang: 'en',
+};

package/lib/routes/decrypt/index.ts

@@ -0,0 +1,115 @@
+import { Route, Data } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+import logger from '@/utils/logger';
+import parser from '@/utils/rss-parser';
+
+export const route: Route = {
+    path: '/',
+    categories: ['finance'],
+    example: '/decrypt',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'News',
+    maintainers: ['pseudoyu'],
+    handler,
+    radar: [
+        {
+            source: ['decrypt.co/'],
+            target: '/',
+        },
+    ],
+    description: 'Get latest news from Decrypt.',
+};
+
+async function handler(ctx): Promise<Data> {
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 20;
+    const rssUrl = 'https://decrypt.co/feed';
+
+    const feed = await parser.parseURL(rssUrl);
+
+    const items = await Promise.all(
+        feed.items
+            .filter((item) => item && item.link && !item.link.includes('/videos'))
+            .slice(0, limit)
+            .map((item) =>
+                cache.tryGet(`decrypt:article:${item.link}`, async () => {
+                    if (!item.link) {
+                        return {};
+                    }
+
+                    try {
+                        const result = await extractFullText(item.link);
+                        return {
+                            title: item.title || 'Untitled',
+                            link: item.link.split('?')[0], // Clean URL by removing query parameters
+                            pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
+                            description: result?.fullText ?? (item.content || ''),
+                            author: item.creator || 'Decrypt',
+                            category: result?.tags ? [...new Set([...(item.categories ?? []), ...result.tags])] : item.categories || [],
+                            guid: item.guid || item.link,
+                            image: result?.featuredImage ?? item.enclosure?.url,
+                        };
+                    } catch (error: any) {
+                        logger.warn(`Couldn't fetch full content for ${item.link}: ${error.message}`);
+
+                        // Fallback to RSS content
+                        return {
+                            title: item.title || 'Untitled',
+                            link: item.link.split('?')[0],
+                            pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
+                            description: item.content || '',
+                            author: item.creator || 'Decrypt',
+                            category: item.categories || [],
+                            guid: item.guid || item.link,
+                            image: item.enclosure?.url,
+                        };
+                    }
+                })
+            )
+    );
+
+    return {
+        title: feed.title || 'Decrypt',
+        link: feed.link || 'https://decrypt.co',
+        description: feed.description || 'Latest news from Decrypt',
+        item: items,
+        language: feed.language || 'en',
+        image: feed.image?.url,
+    } as Data;
+}
+
+async function extractFullText(url: string): Promise<{ fullText: string; featuredImage: string; tags: string[] } | null> {
+    try {
+        const response = await ofetch(url);
+
+        const $ = load(response);
+
+        const nextData = JSON.parse($('script#__NEXT_DATA__').text());
+        const post = nextData.props.pageProps.post;
+
+        if (post.content.length) {
+            const fullText = `<img src="${post.featuredImage.src}" alt="${post.featuredImage.alt}">` + post.content;
+
+            return {
+                fullText,
+                featuredImage: post.featuredImage.src,
+                tags: post.tags.data.map((tag) => tag.name),
+            };
+        }
+
+        return null;
+    } catch (error) {
+        logger.error(`Error extracting full text from ${url}: ${error}`);
+        return null;
+    }
+}

package/lib/routes/decrypt/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Decrypt',
+    url: 'decrypt.co',
+    lang: 'en',
+};

package/lib/routes/discuz/discuz.ts

@@ -1,6 +1,6 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import { load } from 'cheerio';
 import iconv from 'iconv-lite';
 import { parseDate } from '@/utils/parse-date';
@@ -22,14 +22,13 @@ function fixUrl(itemLink, baseUrl) {
 // discuz 7.x 与 discuz x系列 通用文章内容抓取
 async function loadContent(itemLink, charset, header) {
     // 处理编码问题
-    const response = await got({
+    const response = await ofetch.raw(itemLink, {
         method: 'get',
-        url: itemLink,
-        responseType: 'buffer',
+        responseType: 'arrayBuffer',
         headers: header,
     });
 
-    const responseData = iconv.decode(response.data, charset ?? 'utf-8');
+    const responseData = iconv.decode(Buffer.from(response._data), charset ?? 'utf-8');
     if (!responseData) {
         const description = '获取详细内容失败';
         return { description };
@@ -77,14 +76,13 @@ async function handler(ctx) {
         Cookie: cookie,
     };
 
-    const response = await got({
+    const response = await ofetch.raw(link, {
         method: 'get',
-        url: link,
-        responseType: 'buffer',
+        responseType: 'arrayBuffer',
         headers: header,
     });
 
-    const responseData = response.data;
+    const responseData = Buffer.from(response._data);
     // 若没有指定编码，则默认utf-8
     const contentType = response.headers['content-type'] || '';
     let $ = load(iconv.decode(responseData, 'utf-8'));