rsshub 1.0.0-master.f6cb490 → 1.0.0-master.f6f0273

package/lib/routes/swjtu/sports.ts

@@ -0,0 +1,77 @@
+import { Route } from '@/types';
+import cache from '@/utils/cache';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+import { ofetch } from 'ofetch';
+
+const rootURL = 'https://sports.swjtu.edu.cn';
+const pageURL = `${rootURL}/xwzx.htm`;
+
+export const route: Route = {
+    path: '/sports',
+    categories: ['university'],
+    example: '/swjtu/sports',
+    parameters: {},
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['sports.swjtu.edu.cn/'],
+        },
+    ],
+    name: '体育学院',
+    description: '新闻资讯',
+    maintainers: ['AzureG03'],
+    handler,
+};
+
+const getItem = (item, cache) => {
+    const title = item.find('p.toe').text();
+    const link = `${rootURL}/${item.find('a').attr('href')}`;
+
+    return cache.tryGet(link, async () => {
+        const res = await ofetch(link);
+        const $ = load(res);
+
+        const pubDate = parseDate(
+            $('div.info span:nth-of-type(3)')
+                .text()
+                .slice(3)
+                .match(/\d{4}(-|\/|.)\d{1,2}\1\d{1,2}/)?.[0]
+        );
+        const description = $('div.detail-wrap').html();
+        return {
+            title,
+            pubDate,
+            link,
+            description,
+        };
+    });
+};
+
+async function handler() {
+    const res = await ofetch(pageURL);
+
+    const $ = load(res);
+    const $list = $('div.news-list > ul > li');
+
+    const items = await Promise.all(
+        $list.toArray().map((i) => {
+            const $item = $(i);
+            return getItem($item, cache);
+        })
+    );
+
+    return {
+        title: '西南交大体院-新闻资讯',
+        link: pageURL,
+        item: items,
+        allowEmpty: true,
+    };
+}

package/lib/routes/theblock/index.ts

@@ -0,0 +1,142 @@
+import { Route, Data } from '@/types';
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import { load } from 'cheerio';
+import logger from '@/utils/logger';
+
+export const route: Route = {
+    path: '/category/:category',
+    categories: ['finance'],
+    example: '/theblock/category/crypto-ecosystems',
+    parameters: { category: 'News category' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Category',
+    maintainers: ['pseudoyu'],
+    handler,
+    radar: [
+        {
+            source: ['theblock.co/category/:category'],
+            target: '/category/:category',
+        },
+    ],
+    description: 'Get latest news from TheBlock by category. Note that due to website limitations, only article summaries may be available.',
+};
+
+async function handler(ctx): Promise<Data> {
+    const category = ctx.req.param('category');
+    const limit = ctx.req.query('limit') ? Number.parseInt(ctx.req.query('limit')) : 10;
+
+    const apiUrl = `https://www.theblock.co/api/category/${category}`;
+
+    try {
+        const response = await ofetch(apiUrl);
+
+        // Extract articles from the nested data structure
+        const articles = response.data?.articles || [];
+
+        if (!articles.length) {
+            throw new Error(`No articles found for category: ${category}`);
+        }
+
+        const items = await Promise.all(
+            articles.slice(0, limit).map((article) =>
+                cache.tryGet(`theblock:article:${article.url}`, async () => {
+                    try {
+                        // Try to get the full article
+                        const articleResponse = await ofetch(`https://www.theblock.co/api/post/${article.id}/`);
+
+                        const post = articleResponse.post;
+                        const $ = load(post.body, null, false);
+
+                        // If we successfully got the article content
+                        if (post.body.length) {
+                            // Remove unwanted elements
+                            $('.copyright').remove();
+
+                            let fullText = '';
+
+                            if (article.thumbnail) {
+                                fullText += `<p><img src="${post.thumbnail}" alt="${article.title}"></p>`;
+                            }
+                            fullText += post.intro + $.html();
+
+                            if (fullText) {
+                                return {
+                                    title: article.title,
+                                    link: article.url,
+                                    pubDate: parseDate(post.published),
+                                    description: fullText,
+                                    author: article.authors?.map((a) => a.name).join(', ') || 'TheBlock',
+                                    category: [...new Set([post.categories.name, ...post.categories.map((cat) => cat.name), ...post.tags.map((tag) => tag.name)])],
+                                    guid: article.url,
+                                    image: article.thumbnail,
+                                };
+                            }
+                        }
+
+                        // If we couldn't extract specific content, fall back to a summary-based approach
+                        logger.info(`Using summary-based approach for article: ${article.url}`);
+                        return createSummaryItem(article);
+                    } catch (error: any) {
+                        // If we got a 403 error or any other error, use summary approach
+                        logger.warn(`Couldn't fetch full content for ${article.url}: ${error.message}`);
+                        return createSummaryItem(article);
+                    }
+                })
+            )
+        );
+
+        return {
+            title: `TheBlock - ${category.charAt(0).toUpperCase() + category.slice(1).replaceAll('-', ' ')}`,
+            link: `https://www.theblock.co/category/${category}`,
+            item: items,
+            description: `Latest articles from TheBlock in the ${category} category`,
+            language: 'en',
+        } as Data;
+    } catch (error: any) {
+        logger.error(`Error in TheBlock handler: ${error.message}`);
+        throw error;
+    }
+}
+
+// Helper function to create a summary-based item when full content isn't available
+function createSummaryItem(article: any) {
+    let description = '';
+
+    // Add thumbnail if available
+    if (article.thumbnail) {
+        description += `<p><img src="${article.thumbnail}" alt="${article.title}"></p>`;
+    }
+
+    // Add subheading if available
+    if (article.subheading) {
+        description += `<p><strong>${article.subheading}</strong></p>`;
+    }
+
+    // Add preview if available
+    if (article.preview) {
+        description += `<p>${article.preview}</p>`;
+    }
+
+    // Add link to original article
+    description += `<p><a href="${article.url}">Read the full article at TheBlock</a></p>`;
+
+    return {
+        title: article.title,
+        link: article.url,
+        pubDate: parseDate(article.publishedFormatted, 'MMMM D, YYYY, h:mmA [EST]'),
+        description,
+        author: article.authors?.map((a) => a.name).join(', ') || 'TheBlock',
+        category: article.primaryCategory?.name || [],
+        guid: article.url,
+        image: article.thumbnail,
+    };
+}

package/lib/routes/theblock/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'TheBlock',
+    url: 'theblock.co',
+    lang: 'en',
+};

package/lib/routes/theverge/index.ts

@@ -1,8 +1,13 @@
 import { Route } from '@/types';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
+import ofetch from '@/utils/ofetch';
 import parser from '@/utils/rss-parser';
 import { load } from 'cheerio';
+import path from 'node:path';
+import { art } from '@/utils/render';
+import { getCurrentPath } from '@/utils/helpers';
+
+const __dirname = getCurrentPath(import.meta.url);
 
 export const route: Route = {
     path: '/:hub?',
@@ -22,7 +27,7 @@ export const route: Route = {
             source: ['theverge.com/:hub', 'theverge.com/'],
         },
     ],
-    name: 'The Verge',
+    name: 'Category',
     maintainers: ['HenryQW', 'vbali'],
     handler,
     description: `| Hub         | Hub name            |
@@ -43,6 +48,37 @@ export const route: Route = {
   Provides a better reading experience (full text articles) over the official one.`,
 };
 
+const renderBlock = (b) => {
+    switch (b.__typename) {
+        case 'CoreEmbedBlockType':
+            return b.embedHtml;
+        case 'CoreGalleryBlockType':
+            return b.images.map((i) => `<figure><img src="${i.image.thumbnails.horizontal.url.split('?')[0]}" alt="${i.alt}" /><figcaption>${i.caption.html}</figcaption></figure>`).join('');
+        case 'CoreHeadingBlockType':
+            return `<h${b.level}>${b.contents.html}</h${b.level}>`;
+        case 'CoreHTMLBlockType':
+            return b.markup;
+        case 'CoreImageBlockType':
+            return `<figure><img src="${b.thumbnail.url.split('?')[0]}" alt="${b.alt}" /><figcaption>${b.caption.html}</figcaption></figure>`;
+        case 'CoreListBlockType':
+            return `${b.ordered ? '<ol>' : '<ul>'}${b.items.map((i) => `<li>${i.contents.html}</li>`).join('')}${b.ordered ? '</ol>' : '</ul>'}`;
+        case 'CoreParagraphBlockType':
+            return b.contents.html;
+        case 'CorePullquoteBlockType':
+            return `<blockquote>${b.contents.html}</blockquote>`;
+        case 'CoreQuoteBlockType':
+            return `<blockquote>${b.children.map((child) => renderBlock(child)).join('')}</blockquote>`;
+        case 'CoreSeparatorBlockType':
+            return '<hr>';
+        case 'HighlightBlockType':
+            return b.children.map((c) => renderBlock(c)).join('');
+        case 'MethodologyAccordionBlockType':
+            return `<h2>${b.heading.html}</h2>${b.sections.map((s) => `<h3>${s.heading.html}</h3>${s.content.html}`).join('')}`;
+        default:
+            throw new Error(`Unsupported block type: ${b.__typename}`);
+    }
+};
+
 async function handler(ctx) {
     const link = ctx.req.param('hub') ? `https://www.theverge.com/${ctx.req.param('hub')}/rss/index.xml` : 'https://www.theverge.com/rss/index.xml';
 
@@ -51,73 +87,48 @@ async function handler(ctx) {
     const items = await Promise.all(
         feed.items.map((item) =>
             cache.tryGet(item.link, async () => {
-                const response = await got(item.link);
-
-                const $ = load(response.data);
-
-                const content = $('#content');
-                const body = $('.duet--article--article-body-component-container');
-
-                // 处理封面图片
-
-                const cover = $('meta[property="og:image"]');
-
-                if (cover.length > 0) {
-                    $(`<img src=${cover[0].attribs.content}>`).insertBefore(body[0].childNodes[0]);
-                }
+                const response = await ofetch(item.link);
 
-                // 处理封面视频
-                $('div.l-col__main > div.c-video-embed, div.c-entry-hero > div.c-video-embed').each((i, e) => {
-                    const src = `https://volume.vox-cdn.com/embed/${e.attribs['data-volume-uuid']}?autoplay=false`;
+                const $ = load(response);
 
-                    $(`<iframe src="${src}" style="border: 0; top: 0; left: 0; width: 100%; height: 100%; position: absolute;" allowfullscreen scrolling="no"></iframe>`).insertBefore(body[0].childNodes[0]);
-                });
+                const nextData = JSON.parse($('script#__NEXT_DATA__').text());
+                const node = nextData.props.pageProps.hydration.responses.find((x) => x.operationName === 'PostLayoutQuery' || x.operationName === 'StreamLayoutQuery').data.node;
 
-                // 处理封面视频
-                $('div.l-col__main > div.c-video-embed--media iframe').each((i, e) => {
-                    $(e).insertBefore(body[0].childNodes[0]);
+                let description = art(path.join(__dirname, 'templates/header.art'), {
+                    featuredImage: node.featuredImage,
+                    ledeMediaData: node.ledeMediaData,
                 });
 
-                // 处理文章图片
-                content.find('figure.e-image').each((i, e) => {
-                    let src, caption;
-
-                    // 处理 jpeg, png
-                    if ($(e).find('picture > source').length > 0) {
-                        src = $(e)
-                            .find('picture > img')[0]
-                            .attribs.srcset.match(/(?<=320w,).*?(?=520w)/g)[0]
-                            .trim();
-                    } else if ($(e).find('img.c-dynamic-image').length > 0) {
-                        // 处理 gif
-                        src = $(e).find('span.e-image__image')[0].attribs['data-original'];
-                    }
-
-                    // 处理 caption
-                    if ($(e).find('span.e-image__meta').length > 0) {
-                        caption = $(e).find('span.e-image__meta').text();
-                    }
-
-                    const figure = `<figure><img src=${src}>${caption ? `<br><figcaption>${caption}</figcaption>` : ''}</figure>`;
-
-                    $(figure).insertBefore(e);
-
-                    $(e).remove();
-                });
-
-                const lede = $('.duet--article--lede h2:first');
-                if (lede[0]) {
-                    lede.insertBefore(body[0].childNodes[0]);
+                description += node.blocks
+                    .filter((b) => b.__typename !== 'NewsletterBlockType' && b.__typename !== 'RelatedPostsBlockType' && b.__typename !== 'ProductBlockType' && b.__typename !== 'TableOfContentsBlockType')
+                    .map((b) => renderBlock(b))
+                    .join('<br><br>');
+
+                if (node.__typename === 'StreamResourceType') {
+                    description += node.posts.edges
+                        .map(({ node: n }) => {
+                            let d =
+                                `<h2><a href="${n.permalink}">${n.promo.headline || n.title}</a></h2>` +
+                                art(path.join(__dirname, 'templates/header.art'), {
+                                    ledeMediaData: n.ledeMediaData,
+                                });
+                            switch (n.__typename) {
+                                case 'PostResourceType':
+                                    d += n.excerpt.map((e) => e.contents.html).join('<br>');
+                                    break;
+                                case 'QuickPostResourceType':
+                                    d += n.blocks.map((b) => renderBlock(b)).join('<br>');
+                                    break;
+                                default:
+                                    break;
+                            }
+                            return d;
+                        })
+                        .join('<br>');
                 }
 
-                // 移除无用 DOM
-                content.find('.duet--article--comments-join-the-conversation').remove();
-                content.find('.duet--recirculation--related-list').remove();
-                delete item.content;
-                delete item.contentSnippet;
-                delete item.isoDate;
-
-                item.description = body.html();
+                item.description = description;
+                item.category = node.categories;
 
                 return item;
             })

package/lib/routes/theverge/templates/header.art

@@ -0,0 +1,19 @@
+{{ if featuredImage }}
+    <figure>
+        <img src="{{ featuredImage.image.originalUrl.split('?')[0] }}" alt="{{ featuredImage.image.originalUrl.alt }}">
+        <figcaption>{{ featuredImage.image.title }}</figcaption>
+    </figure>
+{{ /if }}
+
+{{ if ledeMediaData }}
+    {{ if ledeMediaData.__typename === 'LedeMediaEmbedType'}}
+        {{@ ledeMediaData.embedHtml }}
+    {{ else if ledeMediaData.__typename === 'LedeMediaImageType' && !featuredImage }}
+        <figure>
+            <img src="{{ ledeMediaData.image.thumbnails.horizontal.url.split('?')[0] }}" alt="{{ ledeMediaData.image.title }}">
+            <figcaption>{{ ledeMediaData.image.credit.plaintext || ledeMediaData.image.title }}</figcaption>
+        </figure>
+    {{ else if ledeMediaData.__typename === 'LedeMediaVideoType'}}
+        <iframe src="https://volume.vox-cdn.com/embed/{{ ledeMediaData.video.volumeUuid }}" allowfullscreen></iframe>
+    {{ /if }}
+{{ /if }}

package/lib/routes/threads/index.ts

@@ -1,10 +1,9 @@
 import { Route, ViewType } from '@/types';
-import ofetch from '@/utils/ofetch';
 import { parseDate } from '@/utils/parse-date';
-import { REPLIES_QUERY, THREADS_QUERY, apiUrl, threadUrl, profileUrl, extractTokens, makeHeader, getUserId, buildContent } from './utils';
-import { destr } from 'destr';
-import cache from '@/utils/cache';
-import { config } from '@/config';
+import { threadUrl, profileUrl, extractTokens, getUserId, buildContent } from './utils';
+import { JSDOM } from 'jsdom';
+import { JSONPath } from 'jsonpath-plus';
+import ofetch from '@/utils/ofetch';
 
 export const route: Route = {
     path: '/:user/:routeParams?',
@@ -25,13 +24,7 @@ Specify options (in the format of query string) in parameter \`routeParams\` to
 | \`showAuthorAvatarInDesc\`       | Show avatar of author in description (RSS body) (Not recommended if your RSS reader extracts images from description)        | \`0\`/\`1\`/\`true\`/\`false\` | \`falseP\`    |
 | \`showEmojiForQuotesAndReply\`   | Use "🔁" instead of "QT", "↩️" instead of "Re"                                                                               | \`0\`/\`1\`/\`true\`/\`false\` | \`true\`      |
 | \`showQuotedInTitle\`            | Show quoted tweet in title                                                                                                   | \`0\`/\`1\`/\`true\`/\`false\` | \`true\`      |
-| \`replies\`                      | Show replies                                                                                                                 | \`0\`/\`1\`/\`true\`/\`false\` | \`true\`      |
-
-Specify different option values than default values to improve readability. The URL
-
-\`\`\`
-https://rsshub.app/threads/zuck/showAuthorInTitle=1&showAuthorInDesc=1&showQuotedAuthorAvatarInDesc=1&showAuthorAvatarInDesc=1&showEmojiForQuotesAndReply=1&showQuotedInTitle=1
-\`\`\``,
+| \`replies\`                      | Show replies                                                                                                                 | \`0\`/\`1\`/\`true\`/\`false\` | \`true\`      |`,
         },
     },
     name: 'User timeline',
@@ -42,10 +35,10 @@ https://rsshub.app/threads/zuck/showAuthorInTitle=1&showAuthorInDesc=1&showQuote
 async function handler(ctx) {
     const { user, routeParams } = ctx.req.param();
     const { lsd } = await extractTokens(user);
-    const userId = await getUserId(user, lsd);
+    const userId = await getUserId(user);
 
     const params = new URLSearchParams(routeParams);
-    const debugJson = {
+    const debugJson: any = {
         params: routeParams,
         lsd,
     };
@@ -60,48 +53,59 @@ async function handler(ctx) {
         replies: params.get('replies') ?? false,
     };
 
-    const threadsResponse = await cache.tryGet(
-        `threads:${userId}:${options.replies}`,
-        () =>
-            ofetch(apiUrl, {
-                method: 'POST',
-                headers: {
-                    ...makeHeader(user, lsd),
-                    'content-type': 'application/x-www-form-urlencoded',
-                },
-                body: new URLSearchParams({
-                    lsd,
-                    variables: JSON.stringify({ userID: userId }),
-                    doc_id: String(options.replies ? REPLIES_QUERY : THREADS_QUERY),
-                }).toString(),
-                parseResponse: (txt) => destr(txt),
-            }),
-        config.cache.routeExpire,
-        false
-    );
+    const response = await ofetch(profileUrl(user), {
+        headers: {
+            'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1',
+            Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
+            'Accept-Encoding': 'gzip, br',
+            'Accept-Language': 'zh-CN,zh;q=0.9',
+            'Cache-Control': 'no-cache',
+            Pragma: 'no-cache',
+            'Sec-Fetch-Dest': 'document',
+            'Sec-Fetch-Mode': 'navigate',
+            'Sec-Fetch-Site': 'none',
+            'Sec-Fetch-User': '?1',
+            'Upgrade-Insecure-Requests': '1',
+        },
+    });
+
+    const dom = new JSDOM(response);
+
+    let threadsData: ThreadItem[] | null = null;
+    for (const el of dom.window.document.querySelectorAll('script[data-sjs]')) {
+        try {
+            const data = JSONPath({
+                path: '$..thread_items[0]',
+                json: JSON.parse(el.textContent || ''),
+            });
+
+            if (data?.length > 0) {
+                threadsData = data as ThreadItem[];
+                break;
+            }
+        } catch {
+            // Skip invalid JSON
+        }
+    }
+
+    if (!threadsData) {
+        throw new Error('Failed to fetch thread data');
+    }
 
     debugJson.profileId = userId;
-    debugJson.response = {
-        response: threadsResponse,
-    };
+    debugJson.response = { response: threadsData };
 
-    const threads = threadsResponse?.data?.mediaData?.threads || [];
-    const userData = threadsResponse?.data?.mediaData?.threads?.[0]?.thread_items?.[0]?.post?.user || {};
-
-    const items = threads.flatMap((thread) =>
-        thread.thread_items
-            .filter((item) => user === item.post.user?.username)
-            .map((item) => {
-                const { title, description } = buildContent(item, options);
-                return {
-                    author: user,
-                    title,
-                    description,
-                    pubDate: parseDate(item.post.taken_at, 'X'),
-                    link: threadUrl(item.post.code),
-                };
-            })
-    );
+    const userData: ThreadUser = threadsData[0]?.post?.user || { username: user, profile_pic_url: '' };
+
+    const items = threadsData
+        .filter((item) => user === item.post.user?.username)
+        .map((item) => ({
+            author: user,
+            title: buildContent(item, options).title,
+            description: buildContent(item, options).description,
+            pubDate: parseDate(item.post.taken_at, 'X'),
+            link: threadUrl(item.post.code),
+        }));
 
     debugJson.items = items;
     ctx.set('json', debugJson);
@@ -110,7 +114,22 @@ async function handler(ctx) {
         title: `${user} (@${user}) on Threads`,
         link: profileUrl(user),
         image: userData?.profile_pic_url,
-        // description: userData.biography,
         item: items,
     };
 }
+
+interface ThreadUser {
+    username: string;
+    profile_pic_url: string;
+}
+
+interface ThreadItem {
+    post: {
+        user?: ThreadUser;
+        taken_at: number;
+        code: string;
+        caption?: {
+            text: string;
+        };
+    };
+}