rsshub 1.0.0-master.ff3a9ce → 1.0.0-master.ff3ea5a

package/lib/config.ts

@@ -37,6 +37,7 @@ export type Config = {
     };
     // proxy
     proxyUri?: string;
+    proxyUris?: string[];
     proxy: {
         protocol?: string;
         host?: string;
@@ -44,6 +45,8 @@ export type Config = {
         auth?: string;
         url_regex: string;
         strategy: 'on_retry' | 'all';
+        failoverTimeout?: number;
+        healthCheckInterval?: number;
     };
     pacUri?: string;
     pacScript?: string;
@@ -73,6 +76,7 @@ export type Config = {
         allow_user_hotlink_template: boolean;
         filter_regex_engine: string;
         allow_user_supply_unsafe_domain: boolean;
+        disable_nsfw: boolean;
     };
     suffix?: string;
     titleLengthLimit: number;
@@ -98,6 +102,7 @@ export type Config = {
         cookies: Record<string, string | undefined>;
         dmImgList?: string;
         dmImgInter?: string;
+        excludeSubtitles?: boolean;
     };
     bitbucket: {
         username?: string;
@@ -216,6 +221,9 @@ export type Config = {
     lightnovel: {
         cookie?: string;
     };
+    lofter: {
+        cookies?: string;
+    };
     lorientlejour: {
         token?: string;
         username?: string;
@@ -255,6 +263,10 @@ export type Config = {
     misskey: {
         accessToken?: string;
     };
+    mixi2: {
+        authToken?: string;
+        authKey?: string;
+    };
     mox: {
         cookie: string;
     };
@@ -348,6 +360,11 @@ export type Config = {
     tsdm39: {
         cookie: string;
     };
+    tumblr: {
+        clientId?: string;
+        clientSecret?: string;
+        refreshToken?: string;
+    };
     twitter: {
         username?: string[];
         password?: string[];
@@ -487,6 +504,11 @@ const calculateValue = () => {
         },
         // proxy
         proxyUri: envs.PROXY_URI,
+        proxyUris: envs.PROXY_URIS
+            ? envs.PROXY_URIS.split(',')
+                  .map((uri) => uri.trim())
+                  .filter(Boolean)
+            : undefined,
         proxy: {
             protocol: envs.PROXY_PROTOCOL,
             host: envs.PROXY_HOST,
@@ -494,6 +516,8 @@ const calculateValue = () => {
             auth: envs.PROXY_AUTH,
             url_regex: envs.PROXY_URL_REGEX || '.*',
             strategy: envs.PROXY_STRATEGY || 'all', // all / on_retry
+            failoverTimeout: toInt(envs.PROXY_FAILOVER_TIMEOUT, 5000),
+            healthCheckInterval: toInt(envs.PROXY_HEALTH_CHECK_INTERVAL, 60000),
         },
         pacUri: envs.PAC_URI,
         pacScript: envs.PAC_SCRIPT,
@@ -524,6 +548,7 @@ const calculateValue = () => {
             allow_user_hotlink_template: toBoolean(envs.ALLOW_USER_HOTLINK_TEMPLATE, false),
             filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2',
             allow_user_supply_unsafe_domain: toBoolean(envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN, false),
+            disable_nsfw: toBoolean(envs.DISABLE_NSFW, false),
         },
         suffix: envs.SUFFIX,
         titleLengthLimit: toInt(envs.TITLE_LENGTH_LIMIT, 150),
@@ -549,6 +574,7 @@ const calculateValue = () => {
             cookies: bilibili_cookies,
             dmImgList: envs.BILIBILI_DM_IMG_LIST,
             dmImgInter: envs.BILIBILI_DM_IMG_INTER,
+            excludeSubtitles: toBoolean(envs.BILIBILI_EXCLUDE_SUBTITLES, false),
         },
         bitbucket: {
             username: envs.BITBUCKET_USERNAME,
@@ -667,6 +693,9 @@ const calculateValue = () => {
         lightnovel: {
             cookie: envs.SECURITY_KEY,
         },
+        lofter: {
+            cookies: envs.LOFTER_COOKIE,
+        },
         lorientlejour: {
             token: envs.LORIENTLEJOUR_TOKEN,
             username: envs.LORIENTLEJOUR_USERNAME,
@@ -706,6 +735,10 @@ const calculateValue = () => {
         misskey: {
             accessToken: envs.MISSKEY_ACCESS_TOKEN,
         },
+        mixi2: {
+            authToken: envs.MIXI2_AUTH_TOKEN,
+            authKey: envs.MIXI2_AUTH_KEY,
+        },
         mox: {
             cookie: envs.MOX_COOKIE,
         },
@@ -799,6 +832,11 @@ const calculateValue = () => {
         tsdm39: {
             cookie: envs.TSDM39_COOKIES,
         },
+        tumblr: {
+            clientId: envs.TUMBLR_CLIENT_ID,
+            clientSecret: envs.TUMBLR_CLIENT_SECRET,
+            refreshToken: envs.TUMBLR_REFRESH_TOKEN,
+        },
         twitter: {
             username: envs.TWITTER_USERNAME?.split(','),
             password: envs.TWITTER_PASSWORD?.split(','),

package/lib/errors/index.tsx

@@ -1,4 +1,5 @@
 import { type NotFoundHandler, type ErrorHandler } from 'hono';
+import { routePath } from 'hono/route';
 import { getDebugInfo, setDebugInfo } from '@/utils/debug-info';
 import { config } from '@/config';
 import * as Sentry from '@sentry/node';
@@ -11,7 +12,7 @@ import { requestMetric } from '@/utils/otel';
 
 export const errorHandler: ErrorHandler = (error, ctx) => {
     const requestPath = ctx.req.path;
-    const matchedRoute = ctx.req.routePath;
+    const matchedRoute = routePath(ctx);
     const hasMatchedRoute = matchedRoute !== '/*';
 
     const debug = getDebugInfo();
@@ -42,7 +43,7 @@ export const errorHandler: ErrorHandler = (error, ctx) => {
         });
     }
 
-    let errorMessage = process.env.NODE_ENV === 'production' ? error.message : error.stack || error.message;
+    let errorMessage = (process.env.NODE_ENV || process.env.VERCEL_ENV) === 'production' ? error.message : error.stack || error.message;
     switch (error.constructor.name) {
         case 'HTTPError':
         case 'RequestError':

package/lib/middleware/debug.ts

@@ -1,4 +1,5 @@
 import { MiddlewareHandler } from 'hono';
+import { routePath } from 'hono/route';
 import { getDebugInfo, setDebugInfo } from '@/utils/debug-info';
 
 const middleware: MiddlewareHandler = async (ctx, next) => {
@@ -17,11 +18,12 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
 
     {
         const debug = getDebugInfo();
-        const hasMatchedRoute = ctx.req.routePath !== '/*';
-        if (!debug.routes[ctx.req.routePath] && hasMatchedRoute) {
-            debug.routes[ctx.req.routePath] = 0;
+        const rPath = routePath(ctx);
+        const hasMatchedRoute = rPath !== '/*';
+        if (!debug.routes[rPath] && hasMatchedRoute) {
+            debug.routes[rPath] = 0;
         }
-        hasMatchedRoute && debug.routes[ctx.req.routePath]++;
+        hasMatchedRoute && debug.routes[rPath]++;
 
         if (ctx.res.headers.get('RSSHub-Cache-Status')) {
             debug.hitCache++;

package/lib/middleware/header.ts

@@ -1,4 +1,5 @@
 import { MiddlewareHandler } from 'hono';
+import { routePath } from 'hono/route';
 import etagCalculate from 'etag';
 import { config } from '@/config';
 import { Data } from '@/types';
@@ -24,9 +25,10 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
     ctx.header('Access-Control-Allow-Origin', config.allowOrigin || new URL(ctx.req.url).host);
 
     await next();
+    const rPath = routePath(ctx);
 
-    if (ctx.req.routePath !== '/*') {
-        ctx.header('X-RSSHub-Route', ctx.req.routePath);
+    if (rPath !== '/*') {
+        ctx.header('X-RSSHub-Route', rPath);
     }
 
     const data: Data = ctx.get('data');

package/lib/registry.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from 'vitest';
+import { describe, expect, it, vi } from 'vitest';
 import app from '@/app';
 import { config } from '@/config';
 
@@ -37,4 +37,27 @@ describe('registry', () => {
         expect(response.headers.get('cache-control')).toBe('no-cache');
         expect(await response.text()).toBe('ok');
     });
+
+    it('namespaces respects DISABLE_NSFW=true', async () => {
+        vi.resetModules();
+        vi.stubEnv('DISABLE_NSFW', 'true');
+
+        const { namespaces } = await import('./registry');
+
+        // All routes in all namespaces should not have nsfw features
+        for (const ns of Object.values(namespaces)) {
+            for (const route of Object.values(ns.routes)) {
+                expect(route.features?.nsfw).not.toBe(true);
+            }
+        }
+        expect(namespaces['2048']).toBeUndefined();
+    });
+
+    it('namespaces includes NSFW routes when DISABLE_NSFW=false', async () => {
+        vi.resetModules();
+        vi.stubEnv('DISABLE_NSFW', 'false');
+
+        const { namespaces } = await import('./registry');
+        expect(namespaces['2048']).toBeDefined();
+    });
 });

package/lib/registry.ts

@@ -1,6 +1,7 @@
 import type { APIRoute, Namespace, Route } from '@/types';
 import { directoryImport } from 'directory-import';
 import { Hono, type Handler } from 'hono';
+import { routePath } from 'hono/route';
 import path from 'node:path';
 import { serveStatic } from '@hono/node-server/serve-static';
 import { config } from '@/config';
@@ -13,16 +14,34 @@ import logger from '@/utils/logger';
 
 const __dirname = import.meta.dirname;
 
+function isSafeRoutes(routes: RoutesType): boolean {
+    return Object.values(routes).every((route: Route) => !route.features?.nsfw);
+}
+
+function safeNamespaces(namespaces: NamespacesType): NamespacesType {
+    const safe: NamespacesType = {};
+
+    for (const [key, value] of Object.entries(namespaces)) {
+        if (value.routes === null || value.routes === undefined || isSafeRoutes(value.routes)) {
+            safe[key] = value;
+        }
+    }
+    return safe;
+}
+
 let modules: Record<string, { route: Route } | { namespace: Namespace }> = {};
-let namespaces: Record<
+
+type RoutesType = Record<
+    string,
+    Route & {
+        location: string;
+    }
+>;
+
+export type NamespacesType = Record<
     string,
     Namespace & {
-        routes: Record<
-            string,
-            Route & {
-                location: string;
-            }
-        >;
+        routes: RoutesType;
         apiRoutes: Record<
             string,
             APIRoute & {
@@ -30,15 +49,21 @@ let namespaces: Record<
             }
         >;
     }
-> = {};
+>;
 
-switch (process.env.NODE_ENV) {
+let namespaces: NamespacesType = {};
+
+switch (process.env.NODE_ENV || process.env.VERCEL_ENV) {
     case 'production':
         namespaces = (await import('../assets/build/routes.js')).default;
         break;
     case 'test':
         // @ts-expect-error
         namespaces = await import('../assets/build/routes.json');
+        if (namespaces.default) {
+            // @ts-ignore
+            namespaces = namespaces.default;
+        }
         break;
     default:
         modules = directoryImport({
@@ -47,6 +72,10 @@ switch (process.env.NODE_ENV) {
         }) as typeof modules;
 }
 
+if (config.feature.disable_nsfw) {
+    namespaces = safeNamespaces(namespaces);
+}
+
 if (Object.keys(modules).length) {
     for (const module in modules) {
         const content = modules[module] as
@@ -158,7 +187,7 @@ for (const namespace in namespaces) {
 
     for (const [path, routeData] of sortedRoutes) {
         const wrappedHandler: Handler = async (ctx) => {
-            logger.debug(`Matched route: ${ctx.req.routePath}`);
+            logger.debug(`Matched route: ${routePath(ctx)}`);
             if (!ctx.get('data')) {
                 if (typeof routeData.handler !== 'function') {
                     if (process.env.NODE_ENV === 'test') {
@@ -222,7 +251,7 @@ if (config.debugInfo) {
 app.use(
     '/*',
     serveStatic({
-        root: './lib/assets',
+        root: path.join(__dirname, 'assets'),
         rewriteRequestPath: (path) => (path === '/favicon.ico' ? '/favicon.png' : path),
     })
 );

package/lib/router.js

@@ -1357,7 +1357,7 @@ router.get('/forum4399/:mtag', lazyloadRouteHandler('./routes/game4399/forum'));
 router.get('/nudt/yjszs/:id?', lazyloadRouteHandler('./routes/universities/nudt/yjszs'));
 
 // dev.to
-router.get('/dev.to/top/:period', lazyloadRouteHandler('./routes/dev.to/top'));
+// router.get('/dev.to/top/:period', lazyloadRouteHandler('./routes/dev.to/top'));
 
 // GameRes 游资网
 router.get('/gameres/hot', lazyloadRouteHandler('./routes/gameres/hot'));

package/lib/routes/121/namespace.ts

@@ -0,0 +1,9 @@
+import type { Namespace } from '@/types';
+
+export const namespace: Namespace = {
+    name: '深圳台风网',
+    url: '121.com.cn',
+    categories: ['forecast'],
+    description: '',
+    lang: 'zh-CN',
+};

package/lib/routes/121/templates/description.art

@@ -0,0 +1,17 @@
+{{ if description }}
+  <p>{{ description }}</p>
+{{ /if }}
+
+{{ if images }}
+  {{ each images image }}
+    {{ if image?.src }}
+      <figure>
+        <img
+          {{ if image.alt }}
+            alt="{{ image.alt }}"
+          {{ /if }}
+        src="{{ image.src }}">
+      </figure>
+    {{ /if }}
+  {{ /each }}
+{{ /if }}

package/lib/routes/121/weather-live.ts

@@ -0,0 +1,104 @@
+import { type Data, type DataItem, type Route, ViewType } from '@/types';
+
+import { art } from '@/utils/render';
+import ofetch from '@/utils/ofetch';
+import { parseDate } from '@/utils/parse-date';
+
+import { type CheerioAPI, load } from 'cheerio';
+import { type Context } from 'hono';
+import path from 'node:path';
+
+export const handler = async (ctx: Context): Promise<Data> => {
+    const limit: number = Number.parseInt(ctx.req.query('limit') ?? '100', 10);
+
+    const baseUrl: string = 'https://tf.121.com.cn';
+    const imgBaseUrl: string = 'https://wx.121.com.cn';
+    const targetUrl: string = new URL('web/weatherLive/', baseUrl).href;
+    const apiUrl: string = new URL('weather/weibo/message.js', baseUrl).href;
+
+    const targetResponse = await ofetch(targetUrl);
+    const $: CheerioAPI = load(targetResponse);
+    const language = $('html').attr('lang') ?? 'zh';
+
+    const response = await ofetch(apiUrl);
+    const messages = await response.text();
+
+    const items: DataItem[] = JSON.parse(messages.split(/var\smessage=/).pop())
+        .slice(0, limit)
+        .map((item): DataItem => {
+            const title: string = item.Title;
+            const description: string | undefined = art(path.join(__dirname, 'templates/description.art'), {
+                description: item.Content,
+                images: item.Img?.map((img: string) => ({
+                    src: new URL(`WeChat/data/weiweb/images/lwspic/${img}`, imgBaseUrl).href,
+                    alt: title,
+                })),
+            });
+            const pubDate: number | string = item.DDatetime;
+            const linkUrl: string | undefined = targetUrl;
+            const guid: string = `121-${title}-${pubDate}`;
+            const image: string | undefined = item.Img?.length > 0 ? new URL(`WeChat/data/weiweb/images/lwspic/${item.Img[0]}`, imgBaseUrl).href : undefined;
+            const updated: number | string = pubDate;
+
+            const processedItem: DataItem = {
+                title,
+                description,
+                pubDate: pubDate ? parseDate(pubDate) : undefined,
+                link: linkUrl ?? new URL(item.id, baseUrl).href,
+                guid,
+                id: guid,
+                content: {
+                    html: description,
+                    text: description,
+                },
+                image,
+                banner: image,
+                updated: updated ? parseDate(updated) : undefined,
+                language,
+            };
+
+            return processedItem;
+        });
+
+    const title: string = $('title').text();
+
+    return {
+        title,
+        description: title,
+        link: targetUrl,
+        item: items,
+        allowEmpty: true,
+        image: $('img').first().attr('src') ? new URL($('img').first().attr('src') as string, baseUrl).href : undefined,
+        author: $('div#webnameDiv').text(),
+        language,
+        id: targetUrl,
+    };
+};
+
+export const route: Route = {
+    path: '/weatherLive',
+    name: '深圳天气直播',
+    url: 'tf.121.com.cn',
+    maintainers: ['nczitzk'],
+    handler,
+    example: '/121/weatherLive',
+    parameters: undefined,
+    description: undefined,
+    categories: ['forecast'],
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportRadar: true,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    radar: [
+        {
+            source: ['tf.121.com.cn', 'tf.121.com.cn/web/weatherLive'],
+            target: '/weatherLive',
+        },
+    ],
+    view: ViewType.Notifications,
+};

package/lib/routes/141jav/index.ts

@@ -47,6 +47,9 @@ export const route: Route = {
 -  \`/141jav/date/2020/07/30\`
 
       \`date\` 类型的关键词必须填写 **日期(年/月/日)**`,
+    features: {
+        nsfw: true,
+    },
 };
 
 async function handler(ctx) {

package/lib/routes/141ppv/index.ts

@@ -47,6 +47,9 @@ export const route: Route = {
 -  \`/141ppv/date/2020/07/30\`
 
       \`date\` 类型的关键词必须填写 **日期(年/月/日)**`,
+    features: {
+        nsfw: true,
+    },
 };
 
 async function handler(ctx) {

package/lib/routes/163/news/rank.ts

@@ -135,7 +135,7 @@ async function handler(ctx) {
     const $ = load(iconv.decode(response.data, 'gbk'));
 
     const list = $('div.tabContents')
-        .eq(timeRange[time].index + (category === 'whole' ? (type === 'click' ? -1 : 2) : (type === 'click' ? 0 : 2)))
+        .eq(timeRange[time].index + (category === 'whole' ? (type === 'click' ? -1 : 2) : type === 'click' ? 0 : 2))
         .find('table tbody tr td a')
         .toArray()
         .map((item) => {

package/lib/routes/18comic/album.ts

@@ -1,10 +1,8 @@
 import { Route } from '@/types';
+import { defaultDomain, getApiUrl, getRootUrl, processApiItems } from './utils';
+import { art } from '@/utils/render';
+import path from 'node:path';
 import cache from '@/utils/cache';
-import got from '@/utils/got';
-import { load } from 'cheerio';
-import { parseDate } from '@/utils/parse-date';
-
-import { defaultDomain, getRootUrl } from './utils';
 
 export const route: Route = {
     path: '/album/:id',
@@ -37,71 +35,68 @@ async function handler(ctx) {
     const id = ctx.req.param('id');
     const { domain = defaultDomain } = ctx.req.query();
     const rootUrl = getRootUrl(domain);
-
     const currentUrl = `${rootUrl}/album/${id}`;
 
-    const response = await got({
-        method: 'get',
-        url: currentUrl,
-    });
-
-    const $ = load(response.data);
+    const apiUrl = `${getApiUrl()}/album?id=${id}`;
 
-    const category = $('span[data-type="tags"]')
-        .first()
-        .find('a')
-        .toArray()
-        .map((c) => $(c).text());
-    const author = $('span[data-type="author"]')
-        .first()
-        .find('a')
-        .toArray()
-        .map((a) => $(a).text())
-        .join(', ');
+    const apiResult = await processApiItems(apiUrl);
 
-    let items = $('.btn-toolbar')
-        .first()
-        .find('a')
-        .toArray()
-        .map((item) => {
-            item = $(item);
-
-            return {
-                title: item.text(),
-                link: `${rootUrl}${item.attr('href')}`,
-                guid: `https://18comic.org${item.attr('href')}`,
-                pubDate: parseDate(item.find('.hidden-xs').text()),
-            };
+    const category = apiResult.tags;
+    const author = apiResult.author.join(', ');
+    const description = apiResult.description;
+    const addTime = apiResult.addtime;
+    let results: any[] = [];
+    if (apiResult.series.length === 0) {
+        results.push({
+            title: apiResult.name,
+            link: `${rootUrl}/photo/${id}`,
+            guid: `${rootUrl}/photo/${id}`,
+            updated: new Date(addTime * 1000),
+            pubDate: new Date(addTime * 1000),
+            category,
+            author,
+            description: art(path.join(__dirname, 'templates/description.art'), {
+                introduction: description,
+                // 不取图片，因为专辑的图片会被分割排序，所以只取封面图
+                images: [`https://cdn-msp3.${domain}/media/albums/${id}_3x4.jpg`],
+                cover: `https://cdn-msp3.${domain}/media/albums/${id}_3x4.jpg`,
+                category,
+            }),
         });
-
-    items = await Promise.all(
-        items.map((item) =>
-            cache.tryGet(item.guid, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-
-                const content = load(detailResponse.data);
-
-                content('.tab-content').remove();
-
-                item.author = author;
-                item.category = category;
-                item.description = `<img src="${content('.thumb-overlay-albums img[data-original]')
-                    .toArray()
-                    .map((image) => content(image).attr('data-original'))
-                    .join('"><img src="')}">`;
-
-                return item;
-            })
-        )
-    );
+    } else {
+        results = await Promise.all(
+            apiResult.series.map((item, index) =>
+                cache.tryGet(`18comic:album:${item.id}`, async () => {
+                    const chapterApiUrl = `${getApiUrl()}/chapter?id=${item.id}`;
+                    const chapterResult = await processApiItems(chapterApiUrl);
+                    const result = {};
+                    const chapterNum = index + 1;
+                    result.title = `第${String(chapterNum)}話 ${item.name === '' ? `${String(chapterNum)}` : item.name}`;
+                    result.link = `${rootUrl}/photo/${item.id}`;
+                    result.guid = `${rootUrl}/photo/${item.id}`;
+                    result.updated = new Date(chapterResult.addtime * 1000);
+                    result.pubDate = addTime;
+                    result.category = category;
+                    result.author = author;
+                    result.description = art(path.join(__dirname, 'templates/description.art'), {
+                        introduction: description,
+                        // 不取图片，因为专辑的图片会被分割排序，所以只取封面图
+                        images: [`https://cdn-msp3.${domain}/media/albums/${item.id}_3x4.jpg`],
+                        cover: `https://cdn-msp3.${domain}/media/albums/${item.id}_3x4.jpg`,
+                        category,
+                    });
+                    return result;
+                })
+            )
+        );
+        results = results.toReversed();
+    }
 
     return {
-        title: $('title').text(),
-        link: currentUrl,
-        item: items,
-        description: $('meta[property="og:description"]').attr('content'),
+        title: `${apiResult.name} - 禁漫天堂`,
+        link: currentUrl.replace(/\?$/, ''),
+        item: results,
+        allowEmpty: true,
+        description,
     };
 }

package/lib/routes/18comic/index.ts

@@ -13,6 +13,7 @@ export const route: Route = {
         supportBT: false,
         supportPodcast: false,
         supportScihub: false,
+        nsfw: true,
     },
     radar: [
         {