rsshub 1.0.0-master.ff3a9ce → 1.0.0-master.ff3ea5a

package/lib/routes/tidb/blog.ts

@@ -0,0 +1,314 @@
+import { type Data, type DataItem, type Route, ViewType } from '@/types';
+
+import cache from '@/utils/cache';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+
+import { type CheerioAPI, load } from 'cheerio';
+import { type Context } from 'hono';
+
+const escapeHtml = (text: string): string => text?.replace(/&/g, '&amp;')?.replace(/</g, '&lt;')?.replace(/>/g, '&gt;')?.replace(/'/g, '&quot;')?.replace(/'/g, '&#039;') ?? text;
+
+const parseTextChildren = (children: any[]): string => children.map((child: any) => escapeHtml(child.text)).join('');
+
+const parseImageNode = (node: any): string => {
+    const titleAttr = node.title ? ` title="${escapeHtml(node.title)}"` : '';
+    const altAttr = node.alt ? ` alt="${escapeHtml(node.alt)}"` : '';
+    const styleAttr = node.size ? ` style="width:${node.size.width}px;height:${node.size.height}px;"` : '';
+    return `<img src="${escapeHtml(node.url)}"${titleAttr}${altAttr}${styleAttr}>`;
+};
+
+const parseListItemNode = (listItem: any): string => `<li>${parseContentToHtml(listItem.children)}</li>`;
+
+const parseListNode = (node: any): string => {
+    const tag = node.ordered ? 'ol' : 'ul';
+    const startAttr = node.ordered && node.start !== 1 ? ` start="${node.start}"` : '';
+    const listItemsHtml = node.children.map((item: any) => parseListItemNode(item)).join('');
+    return `<${tag}${startAttr}>${listItemsHtml}</${tag}>`;
+};
+
+const parseParagraphChildren = (children: any[]): string =>
+    children
+        .map((child: any) => {
+            if (child.text !== undefined) {
+                return escapeHtml(child.text);
+            } else if (child.type === 'image') {
+                return parseImageNode(child);
+            }
+            return '';
+        })
+        .join('');
+
+const parseContentToHtml = (content: any[]): string =>
+    content
+        ?.map((node: any) => {
+            switch (node.type) {
+                case 'paragraph':
+                    return `<p>${parseParagraphChildren(node.children)}</p>`;
+                case 'image':
+                    return parseImageNode(node);
+                case 'heading':
+                    return `<h${node.depth}>${parseTextChildren(node.children)}</h${node.depth ?? ''}>`;
+                case 'code':
+                    return `<pre><code${node.lang ? ` class="language-${node.lang}"` : ''}>${parseTextChildren(node.children)}</code></pre>`;
+                case 'list':
+                    return parseListNode(node);
+                case 'blockquote':
+                    return `<blockquote>${parseContentToHtml(node.children)}</blockquote>`;
+                default:
+                    return '';
+            }
+        })
+        .join('') ?? '';
+
+export const handler = async (ctx: Context): Promise<Data> => {
+    const { category = 'latest' } = ctx.req.param();
+    const limit: number = Number.parseInt(ctx.req.query('limit') ?? '20', 10);
+
+    const baseUrl: string = 'https://tidb.net';
+    const targetUrl: string = new URL(`blog${category === 'latest' ? '' : `/c/${category}`}`, baseUrl).href;
+    const targetResponse = await ofetch(targetUrl);
+
+    const buildId: string | undefined = targetResponse.match(/"buildId":"(.*?)"/)?.[1];
+
+    if (!buildId) {
+        throw new Error('Build ID not found.');
+    }
+
+    const $: CheerioAPI = load(targetResponse);
+    const language = $('html').attr('lang') ?? 'zh';
+
+    const apiUrl: string = new URL(`_next/data/${buildId}/${language}/blog${category === 'latest' ? '' : `/c/${category}`}.json`, baseUrl).href;
+
+    const response = await ofetch(apiUrl, {
+        query: {
+            latest: true,
+        },
+    });
+
+    let items: DataItem[] = [];
+
+    items = response.pageProps.blogs.content.slice(0, limit).map((item): DataItem => {
+        const title: string = item.title;
+        const description: string | undefined = item.summary;
+        const pubDate: number | string = item.publishedAt;
+        const linkUrl: string | undefined = item.slug ? `blog/${item.slug}` : undefined;
+        const categories: string[] = [...new Set([item.category?.name, ...(item.tags ?? []).map((c) => c.name)].filter(Boolean))];
+        const authors: DataItem['author'] = item.author?.username
+            ? [
+                  {
+                      name: item.author.username,
+                      url: new URL(`u/${item.author.username}`, baseUrl).href,
+                      avatar: item.author.avatarURL,
+                  },
+              ]
+            : undefined;
+        const guid: string = item.slug ?? '';
+        const updated: number | string = item.lastModifiedAt ?? pubDate;
+
+        const processedItem: DataItem = {
+            title,
+            description,
+            pubDate: pubDate ? parseDate(pubDate) : undefined,
+            link: linkUrl ? new URL(linkUrl, baseUrl).href : undefined,
+            category: categories,
+            author: authors,
+            guid,
+            id: guid,
+            content: {
+                html: description,
+                text: description,
+            },
+            updated: updated ? parseDate(updated) : undefined,
+            language,
+        };
+
+        return processedItem;
+    });
+
+    items = await Promise.all(
+        items.map((item) => {
+            if (!item.link) {
+                return item;
+            }
+
+            return cache.tryGet(item.link, async (): Promise<DataItem> => {
+                const detailUrl: string = new URL(`blog/api/posts/${item.guid}/detail`, baseUrl).href;
+
+                const detailResponse = await ofetch(detailUrl, {
+                    query: {
+                        visit: true,
+                    },
+                });
+
+                const title: string = detailResponse.title;
+                const description: string | undefined = detailResponse.content ? parseContentToHtml(JSON.parse(detailResponse.content)) : item.description;
+                const pubDate: number | string = detailResponse.publishedAt;
+                const linkUrl: string | undefined = `blog/${detailResponse.slug}`;
+                const categories: string[] = [...new Set([detailResponse.category?.name, ...(detailResponse.tags ?? []).map((c) => c.name)].filter(Boolean))];
+                const authors: DataItem['author'] = detailResponse.author?.username
+                    ? [
+                          {
+                              name: detailResponse.author.username,
+                              url: new URL(`u/${detailResponse.author.username}`, baseUrl).href,
+                              avatar: detailResponse.author.avatarURL,
+                          },
+                      ]
+                    : undefined;
+                const guid: string = `tidb-blog-${detailResponse.slug}`;
+                const updated: number | string = detailResponse.lastModifiedAt ?? pubDate;
+
+                const processedItem: DataItem = {
+                    title,
+                    description,
+                    pubDate: pubDate ? parseDate(pubDate) : undefined,
+                    link: new URL(linkUrl, baseUrl).href,
+                    category: categories,
+                    author: authors,
+                    guid,
+                    id: guid,
+                    content: {
+                        html: description,
+                        text: description,
+                    },
+                    updated: updated ? parseDate(updated) : undefined,
+                    language,
+                };
+
+                return {
+                    ...item,
+                    ...processedItem,
+                };
+            });
+        })
+    );
+
+    const title: string = $('title').text();
+
+    return {
+        title,
+        description: $('meta[property="og:description"]').attr('content'),
+        link: targetUrl,
+        item: items,
+        allowEmpty: true,
+        image: $('meta[property="og:image"]').attr('content'),
+        author: title.split(/\|/).pop()?.trim(),
+        language,
+        id: targetUrl,
+    };
+};
+
+export const route: Route = {
+    path: '/blog/c/:category?',
+    name: '专栏分类',
+    url: 'tidb.net',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/tidb/blog/c/latest',
+    parameters: {
+        category: {
+            description: '分类，默认为 `latest`，即全部文章，可在对应分类页 URL 中找到',
+            options: [
+                {
+                    label: '全部文章',
+                    value: 'latest',
+                },
+                {
+                    label: '管理与运维',
+                    value: 'management-and-operation',
+                },
+                {
+                    label: '实践案例',
+                    value: 'practical-case',
+                },
+                {
+                    label: '架构选型',
+                    value: 'architecture-selection',
+                },
+                {
+                    label: '原理解读',
+                    value: 'principle-interpretation',
+                },
+                {
+                    label: '应用开发',
+                    value: 'application-development',
+                },
+                {
+                    label: '社区动态',
+                    value: 'community-feeds',
+                },
+            ],
+        },
+    },
+    description: `:::tip
+订阅 [管理与运维](https://tidb.net/blog/c/management-and-operation)，其源网址为 \`https://tidb.net/blog/c/management-and-operation\`，请参考该 URL 指定部分构成参数，此时路由为 [\`/tidb/blog/c/management-and-operation\`](https://rsshub.app/tidb/blog/c/management-and-operation)。
+:::
+
+| 分类                                                           | ID                                                                                  |
+| -------------------------------------------------------------- | ----------------------------------------------------------------------------------- |
+| [全部文章](https://tidb.net/blog)                              | [latest](https://rsshub.app/tidb/blog)                                              |
+| [管理与运维](https://tidb.net/blog/c/management-and-operation) | [management-and-operation](https://rsshub.app/tidb/blog/c/management-and-operation) |
+| [实践案例](https://tidb.net/blog/c/practical-case)             | [practical-case](https://rsshub.app/tidb/blog/c/practical-case)                     |
+| [架构选型](https://tidb.net/blog/c/architecture-selection)     | [architecture-selection](https://rsshub.app/tidb/blog/c/architecture-selection)     |
+| [原理解读](https://tidb.net/blog/c/principle-interpretation)   | [principle-interpretation](https://rsshub.app/tidb/blog/c/principle-interpretation) |
+| [应用开发](https://tidb.net/blog/c/application-development)    | [application-development](https://rsshub.app/tidb/blog/c/application-development)   |
+| [社区动态](https://tidb.net/blog/c/community-feeds)            | [community-feeds](https://rsshub.app/tidb/blog/c/community-feeds)                   |
+
+`,
+    categories: ['programming'],
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['tidb.net/blog', 'tidb.net/blog/c/:category'],
+            target: (params) => {
+                const category: string = params.category;
+
+                return `/tidb/blog/c${category ? `/${category}` : ''}`;
+            },
+        },
+        {
+            title: '全部文章',
+            source: ['tidb.net/blog'],
+            target: '/blog/c/latest',
+        },
+        {
+            title: '管理与运维',
+            source: ['tidb.net/blog/c/management-and-operation'],
+            target: '/blog/c/management-and-operation',
+        },
+        {
+            title: '实践案例',
+            source: ['tidb.net/blog/c/practical-case'],
+            target: '/blog/c/practical-case',
+        },
+        {
+            title: '架构选型',
+            source: ['tidb.net/blog/c/architecture-selection'],
+            target: '/blog/c/architecture-selection',
+        },
+        {
+            title: '原理解读',
+            source: ['tidb.net/blog/c/principle-interpretation'],
+            target: '/blog/c/principle-interpretation',
+        },
+        {
+            title: '应用开发',
+            source: ['tidb.net/blog/c/application-development'],
+            target: '/blog/c/application-development',
+        },
+        {
+            title: '社区动态',
+            source: ['tidb.net/blog/c/community-feeds'],
+            target: '/blog/c/community-feeds',
+        },
+    ],
+    view: ViewType.Articles,
+};

package/lib/routes/tidb/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'TiDB 社区',
+    url: 'tidb.net',
+    categories: ['programming'],
+    description: '',
+    lang: 'zh-CN',
+};

package/lib/routes/tumblr/namespace.ts

@@ -0,0 +1,17 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'Tumblr',
+    url: 'tumblr.com',
+    lang: 'en',
+    description: `Register an application on \`https://www.tumblr.com/oauth/apps\`.
+
+- \`TUMBLR_CLIENT_ID\`: The key is labelled as \`OAuth consumer Key\` in the info page of the registered application.
+- \`TUMBLR_CLIENT_SECRET\`: The key is labelled as \`OAuth consumer Secret\` in the info page of the registered application.
+- \`TUMBLR_REFRESH_TOKEN\`: Navigate to \`https://www.tumblr.com/oauth2/authorize?client_id=\${CLIENT_ID}&response_type=code&scope=basic%20offline_access&state=mystate\` in your browser and login. After doing so, you'll be redirected to the URL you defined when registering the application. Look for the \`code\` parameter in the URL. You can then call \`curl -F grant_type=authorization_code -F "code=\${CODE}" -F "client_id=\${CLIENT_ID}" -F "client_secret=\${CLIENT_SECRET}" "https://api.tumblr.com/v2/oauth2/token"\`
+
+Two login methods are currently supported:
+
+- \`TUMBLR_CLIENT_ID\`: The key never expires, however blogs that are "dashboard only" cannot be accessed.
+- \`TUMBLR_CLIENT_ID\` + \`TUMBLR_CLIENT_SECRET\` + \`TUMBLR_REFRESH_TOKEN\`: The refresh token will expire and will need to be regenerated, "dashboard only" blogs can be accessed.`,
+};

package/lib/routes/tumblr/posts.ts

@@ -0,0 +1,74 @@
+import { Data, Route } from '@/types';
+import got from '@/utils/got';
+import utils from './utils';
+import type { Context } from 'hono';
+import { config } from '@/config';
+import ConfigNotFoundError from '@/errors/types/config-not-found';
+import { fallback, queryToInteger } from '@/utils/readable-social';
+
+export const route: Route = {
+    path: '/posts/:blog',
+    categories: ['blog'],
+    example: '/tumblr/posts/biketouring-nearby',
+    parameters: {
+        blog: 'Blog identifier (see `https://www.tumblr.com/docs/en/api/v2#blog-identifiers`)',
+    },
+    radar: [],
+    features: {
+        requireConfig: [
+            {
+                name: 'TUMBLR_CLIENT_ID',
+                description: 'Please see above for details.',
+            },
+            {
+                name: 'TUMBLR_CLIENT_SECRET',
+                description: 'Please see above for details.',
+            },
+            {
+                name: 'TUMBLR_REFRESH_TOKEN',
+                description: 'Please see above for details.',
+            },
+        ],
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: 'Posts',
+    maintainers: ['Rakambda'],
+    description: `::: tip
+Tumblr provides official RSS feeds for non "dashboard only" blogs, for instance [https://biketouring-nearby.tumblr.com](https://biketouring-nearby.tumblr.com/rss).
+:::`,
+    handler,
+};
+
+async function handler(ctx: Context): Promise<Data> {
+    if (!config.tumblr || !config.tumblr.clientId) {
+        throw new ConfigNotFoundError('Tumblr RSS is disabled due to the lack of <a href="https://docs.rsshub.app/deploy/config#route-specific-configurations">relevant config</a>');
+    }
+
+    const blogIdentifier = ctx.req.param('blog');
+    const limit = fallback(undefined, queryToInteger(ctx.req.query('limit')), 20);
+
+    const response = await got.get(`https://api.tumblr.com/v2/blog/${blogIdentifier}/posts`, {
+        searchParams: {
+            ...utils.generateAuthParams(),
+            limit,
+        },
+        headers: await utils.generateAuthHeaders(),
+    });
+
+    const blog = response.data.response.blog;
+    const posts = response.data.response.posts.map((post: any) => utils.processPost(post));
+
+    return {
+        title: `Tumblr - ${blogIdentifier} - Posts`,
+        author: blog?.name,
+        link: blog?.url ?? `https://${blogIdentifier}/`,
+        item: posts,
+        allowEmpty: true,
+        image: blog?.avatar?.slice(-1)?.url,
+        description: blog?.description,
+    };
+}

package/lib/routes/tumblr/utils.ts

@@ -0,0 +1,110 @@
+import { parseDate } from '@/utils/parse-date';
+import { DataItem } from '@/types';
+import { config } from '@/config';
+import cache from '@/utils/cache';
+import logger from '@/utils/logger';
+import got from '@/utils/got';
+
+const getAccessToken: () => Promise<string | null> = async () => {
+    let accessToken: string | null = await cache.get('tumblr:accessToken', false);
+    if (!accessToken) {
+        try {
+            const newAccessToken = await tokenRefresher();
+            if (newAccessToken) {
+                accessToken = newAccessToken;
+            }
+        } catch (error) {
+            // Return the `accessToken=null` value to indicate that the token is not available. Calls will only use the `apiKey` as a fallback to maybe hit non "dashborad only" blogs.
+            logger.error('Failed to refresh Tumblr token, using only client id as fallback', error);
+        }
+    }
+    return accessToken;
+};
+
+const generateAuthHeaders: () => Promise<{ Authorization?: string }> = async () => {
+    const accessToken = await getAccessToken();
+    if (!accessToken) {
+        return {};
+    }
+    return {
+        Authorization: `Bearer ${accessToken}`,
+    };
+};
+
+const generateAuthParams: () => { apiKey?: string } = () => ({
+    apiKey: config.tumblr.clientId,
+});
+
+const processPost: (post: any) => DataItem = (post) => {
+    let description = '';
+
+    switch (post.type) {
+        case 'text':
+            description = post.body;
+            break;
+        case 'photo':
+            for (const photo of post.photos ?? []) {
+                description += `<img src="${photo.original_size.url}"/><br/>`;
+            }
+            break;
+        case 'link':
+            description = post.url;
+            break;
+        case 'audio':
+            description = post.embed;
+            break;
+        default:
+            break;
+    }
+
+    return {
+        id: post.id_string,
+        title: post.summary ?? `New post from ${post.blog_name}`,
+        link: post.post_url,
+        pubDate: parseDate(post.timestamp * 1000),
+        category: post.tags,
+        description,
+    };
+};
+
+let tokenRefresher: () => Promise<string | null> = () => Promise.resolve(null);
+if (config.tumblr && config.tumblr.clientId && config.tumblr.clientSecret && config.tumblr.refreshToken) {
+    tokenRefresher = async (): Promise<string | null> => {
+        let refreshToken = config.tumblr.refreshToken;
+
+        // Restore already refreshed tokens
+        const previousRefreshTokenSerialized = await cache.get('tumblr:refreshToken', false);
+        if (previousRefreshTokenSerialized) {
+            const previousRefreshToken = JSON.parse(previousRefreshTokenSerialized);
+            if (previousRefreshToken.startToken === refreshToken) {
+                refreshToken = previousRefreshToken.currentToken;
+            }
+        }
+        const response = await got.post('https://api.tumblr.com/v2/oauth2/token', {
+            form: {
+                grant_type: 'refresh_token',
+                client_id: config.tumblr.clientId,
+                client_secret: config.tumblr.clientSecret,
+                refresh_token: refreshToken,
+            },
+        });
+        if (!response.data?.access_token || !response.data?.refresh_token) {
+            return null;
+        }
+        const accessToken = response.data.access_token;
+        const newRefreshToken = response.data.refresh_token;
+        const expiresIn = response.data.expires_in;
+
+        // Access tokens expire after 42 minutes, remove 30 seconds to renew the token before it expires (to avoid making a request right when it ends).
+        await cache.set('tumblr:accessToken', accessToken, (expiresIn ?? 2520) - 30);
+        // Store the new refresh token associated with the one that was provided first.
+        // We may be able to restore the new token if the app is restarted. This will avoid reusing the old token and have a failing request.
+        // Keep it for a year (not clear how long the refresh token lasts).
+        const cacheEntry = { startToken: config.tumblr.refreshToken, currentToken: newRefreshToken };
+        await cache.set(`tumblr:refreshToken`, JSON.stringify(cacheEntry), 31_536_000);
+
+        return accessToken;
+    };
+}
+
+export default { processPost, generateAuthParams, generateAuthHeaders };

package/lib/routes/tver/namespace.ts

@@ -0,0 +1,7 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: 'TVer',
+    url: 'tver.jp',
+    lang: 'ja',
+};

package/lib/routes/tver/series.ts

@@ -0,0 +1,98 @@
+import { Route, Data, DataItem } from '@/types';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+import timezone from '@/utils/timezone';
+import { Context } from 'hono';
+
+export const route: Route = {
+    path: '/series/:id',
+    categories: ['traditional-media'],
+    example: '/tver/series/srx2o7o3c8',
+    parameters: {
+        id: 'Series ID (as it appears in URLs). For example, in https://tver.jp/series/srx2o7o3c8, the ID is "srx2o7o3c8".',
+    },
+    radar: [
+        {
+            source: ['tver.jp/series/:id'],
+            target: '/series/:id',
+        },
+    ],
+    name: 'Series',
+    maintainers: ['yuikisaito'],
+    handler,
+};
+
+const commonHeaders = {
+    Accept: '*/*',
+    'Accept-Language': 'ja,en-US;q=0.7,en;q=0.3',
+    'Cache-Control': 'no-cache',
+    Pragma: 'no-cache',
+    'Sec-GPC': '1',
+    'Sec-Fetch-Dest': 'empty',
+    'Sec-Fetch-Mode': 'cors',
+    'Sec-Fetch-Site': 'same-site',
+};
+
+async function handler(ctx: Context): Promise<Data> {
+    const { id } = ctx.req.param();
+
+    const { result: browser } = await ofetch('https://platform-api.tver.jp/v2/api/platform_users/browser/create', {
+        method: 'POST',
+        body: 'device_type=pc',
+        headers: {
+            ...commonHeaders,
+            'Content-Type': 'application/x-www-form-urlencoded',
+        },
+        referrer: 'https://s.tver.jp/',
+        credentials: 'omit',
+        mode: 'cors',
+    });
+
+    const { platform_uid, platform_token } = browser;
+
+    const { title, description, broadcastProvider } = await ofetch(`https://statics.tver.jp/content/series/${id}.json`, {
+        method: 'GET',
+        headers: {
+            ...commonHeaders,
+        },
+        referrer: 'https://tver.jp/',
+        credentials: 'omit',
+        mode: 'cors',
+    });
+
+    const { result } = await ofetch(`https://platform-api.tver.jp/service/api/v1/callSeriesEpisodes/${id}?platform_uid=${platform_uid}&platform_token=${platform_token}`, {
+        method: 'GET',
+        headers: {
+            ...commonHeaders,
+            'x-tver-platform-type': 'web',
+        },
+        referrer: 'https://tver.jp/',
+        credentials: 'omit',
+        mode: 'cors',
+    });
+
+    const items: DataItem[] = (result.contents?.[0]?.contents ?? [])
+        .filter((i) => i.type === 'episode')
+        .map((i) => {
+            const rawPubDate = i.content.broadcastDateLabel;
+            const cleanedPubDate = rawPubDate.replaceAll(/\(.*?\)|放送分/g, '').trim();
+            const parsedPubDate = timezone(parseDate(cleanedPubDate, 'M月D日'), +9).toDateString();
+
+            return {
+                title: i.content.title,
+                link: `https://tver.jp/episodes/${i.content.id}`,
+                image: `https://statics.tver.jp/images/content/thumbnail/episode/xlarge/${i.content.id}.jpg`,
+                pubDate: parsedPubDate,
+            };
+        });
+
+    return {
+        title: 'TVer - ' + title,
+        description,
+        author: broadcastProvider.name,
+        link: `https://tver.jp/series/${id}`,
+        image: `https://statics.tver.jp/images/content/thumbnail/series/xlarge/${id}.jpg`,
+        language: 'ja',
+        item: items,
+    };
+}

package/lib/routes/twitter/api/web-api/login.ts

@@ -28,9 +28,7 @@ async function login({ username, password, authenticationSecret }) {
         await loginLimiterQueue.removeTokens(1);
 
         const cookieJar = new CookieJar();
-        const browser = await puppeteer({
-            stealth: true,
-        });
+        const browser = await puppeteer();
         const page = await browser.newPage();
         await page.goto('https://x.com/i/flow/login');
         await page.waitForSelector('input[autocomplete="username"]');