rsshub 1.0.0-master.f75997f → 1.0.0-master.f764bde

package/lib/config.ts

@@ -1,16 +1,18 @@
-import 'dotenv/config';
 import randUserAgent from '@/utils/rand-user-agent';
+import 'dotenv/config';
 import { ofetch } from 'ofetch';
 
 let envs = process.env;
 
 export type Config = {
+    // app config
     disallowRobot: boolean;
     enableCluster?: string;
     isPackage: boolean;
     nodeName?: string;
     puppeteerWSEndpoint?: string;
     chromiumExecutablePath?: string;
+    // network
     connect: {
         port: number;
     };
@@ -20,6 +22,7 @@ export type Config = {
     ua: string;
     trueUA: string;
     allowOrigin?: string;
+    // cache
     cache: {
         type: string;
         requestTimeout: number;
@@ -32,6 +35,7 @@ export type Config = {
     redis: {
         url: string;
     };
+    // proxy
     proxyUri?: string;
     proxy: {
         protocol?: string;
@@ -39,19 +43,27 @@ export type Config = {
         port?: string;
         auth?: string;
         url_regex: string;
+        strategy: 'on_retry' | 'all';
     };
-    proxyStrategy: string;
     pacUri?: string;
     pacScript?: string;
+    // access control
     accessKey?: string;
+    // logging
     debugInfo: string;
     loggerLevel: string;
     noLogfiles?: boolean;
+    otel: {
+        seconds_bucket?: string;
+        milliseconds_bucket?: string;
+    };
     showLoggerTimestamp?: boolean;
     sentry: {
         dsn?: string;
         routeTimeout: number;
     };
+    enableRemoteDebugging?: boolean;
+    // feed config
     hotlink: {
         template?: string;
         includePaths?: string[];
@@ -70,11 +82,16 @@ export type Config = {
         temperature?: number;
         maxTokens?: number;
         endpoint: string;
-        prompt?: string;
+        inputOption: string;
+        promptTitle: string;
+        promptDescription: string;
     };
+
+    // Route-specific Configurations
     bilibili: {
         cookies: Record<string, string | undefined>;
         dmImgList?: string;
+        dmImgInter?: string;
     };
     bitbucket: {
         username?: string;
@@ -87,9 +104,15 @@ export type Config = {
     bupt: {
         portal_cookie?: string;
     };
+    caixin: {
+        cookie?: string;
+    };
     civitai: {
         cookie?: string;
     };
+    dianping: {
+        cookie?: string;
+    };
     dida365: {
         username?: string;
         password?: string;
@@ -144,6 +167,9 @@ export type Config = {
     google: {
         fontsApiKey?: string;
     };
+    guozaoke: {
+        cookies?: string;
+    };
     hefeng: {
         key?: string;
     };
@@ -154,7 +180,6 @@ export type Config = {
         username?: string;
         password?: string;
         bearertoken?: string;
-        iap_receipt?: string;
     };
     instagram: {
         username?: string;
@@ -169,12 +194,25 @@ export type Config = {
     javdb: {
         session?: string;
     };
+    keylol: {
+        cookie?: string;
+    };
     lastfm: {
         api_key?: string;
     };
     lightnovel: {
         cookie?: string;
     };
+    lorientlejour: {
+        token?: string;
+        username?: string;
+        password?: string;
+    };
+    malaysiakini: {
+        email?: string;
+        password?: string;
+        refreshToken?: string;
+    };
     manhuagui: {
         cookie?: string;
     };
@@ -214,6 +252,9 @@ export type Config = {
     notion: {
         key?: string;
     };
+    patreon: {
+        sessionId?: string;
+    };
     pianyuan: {
         cookie?: string;
     };
@@ -233,6 +274,9 @@ export type Config = {
     qingting: {
         id?: string;
     };
+    readwise: {
+        accessToken?: string;
+    };
     saraba1st: {
         cookie?: string;
     };
@@ -245,24 +289,51 @@ export type Config = {
     scihub: {
         host?: string;
     };
+    sis001: {
+        baseUrl?: string;
+    };
+    skeb: {
+        bearerToken?: string;
+    };
+    sorrycc: {
+        cookie?: string;
+    };
     spotify: {
         clientId?: string;
         clientSecret?: string;
         refreshToken?: string;
     };
+    sspai: {
+        bearertoken?: string;
+    };
     telegram: {
         token?: string;
+        session?: string;
+        apiId?: number;
+        apiHash?: string;
+        maxConcurrentDownloads?: number;
+        proxy?: {
+            host?: string;
+            port?: number;
+            secret?: string;
+        };
     };
     tophub: {
         cookie?: string;
     };
+    tsdm39: {
+        cookie: string;
+    };
     twitter: {
-        oauthTokens?: string[];
-        oauthTokenSecrets?: string[];
-        username?: string;
-        password?: string;
-        authenticationSecret?: string;
-        cookie?: string;
+        username?: string[];
+        password?: string[];
+        authenticationSecret?: string[];
+        phoneOrEmail?: string[];
+        authToken?: string[];
+    };
+    uestc: {
+        bbsCookie?: string;
+        bbsAuthStr?: string;
     };
     weibo: {
         app_key?: string;
@@ -280,12 +351,23 @@ export type Config = {
         device_id?: string;
         refresh_token?: string;
     };
+    xiaohongshu: {
+        cookie?: string;
+    };
     ximalaya: {
         token?: string;
     };
+    xsijishe: {
+        cookie?: string;
+        userAgent?: string;
+    };
     xueqiu: {
         cookies?: string;
     };
+    yamibo: {
+        salt?: string;
+        auth?: string;
+    };
     youtube: {
         key?: string;
         clientId?: string;
@@ -360,7 +442,6 @@ const calculateValue = () => {
         requestTimeout: toInt(envs.REQUEST_TIMEOUT, 30000), // Milliseconds to wait for the server to end the response before aborting the request
         ua: envs.UA ?? (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' })),
         trueUA: TRUE_UA,
-        // cors request
         allowOrigin: envs.ALLOW_ORIGIN,
         // cache
         cache: {
@@ -384,8 +465,8 @@ const calculateValue = () => {
             port: envs.PROXY_PORT,
             auth: envs.PROXY_AUTH,
             url_regex: envs.PROXY_URL_REGEX || '.*',
+            strategy: envs.PROXY_STRATEGY || 'all', // all / on_retry
         },
-        proxyStrategy: envs.PROXY_STRATEGY || 'all', // all / on_retry
         pacUri: envs.PAC_URI,
         pacScript: envs.PAC_SCRIPT,
         // access control
@@ -395,11 +476,16 @@ const calculateValue = () => {
         debugInfo: envs.DEBUG_INFO || 'true',
         loggerLevel: envs.LOGGER_LEVEL || 'info',
         noLogfiles: toBoolean(envs.NO_LOGFILES, false),
+        otel: {
+            seconds_bucket: envs.OTEL_SECONDS_BUCKET || '0.01,0.1,1,2,5,15,30,60',
+            milliseconds_bucket: envs.OTEL_MILLISECONDS_BUCKET || '10,20,50,100,250,500,1000,5000,15000',
+        },
         showLoggerTimestamp: toBoolean(envs.SHOW_LOGGER_TIMESTAMP, false),
         sentry: {
             dsn: envs.SENTRY,
             routeTimeout: toInt(envs.SENTRY_ROUTE_TIMEOUT, 30000),
         },
+        enableRemoteDebugging: toBoolean(envs.ENABLE_REMOTE_DEBUGGING, false),
         // feed config
         hotlink: {
             template: envs.HOTLINK_TEMPLATE,
@@ -419,13 +505,16 @@ const calculateValue = () => {
             temperature: toInt(envs.OPENAI_TEMPERATURE, 0.2),
             maxTokens: toInt(envs.OPENAI_MAX_TOKENS, 0) || undefined,
             endpoint: envs.OPENAI_API_ENDPOINT || 'https://api.openai.com/v1',
-            prompt: envs.OPENAI_PROMPT || 'Please summarize the following article and reply with markdown format.',
+            inputOption: envs.OPENAI_INPUT_OPTION || 'description',
+            promptDescription: envs.OPENAI_PROMPT || 'Please summarize the following article and reply with markdown format.',
+            promptTitle: envs.OPENAI_PROMPT_TITLE || 'Please translate the following title into Simplified Chinese and reply only translated text.',
         },
 
         // Route-specific Configurations
         bilibili: {
             cookies: bilibili_cookies,
             dmImgList: envs.BILIBILI_DM_IMG_LIST,
+            dmImgInter: envs.BILIBILI_DM_IMG_INTER,
         },
         bitbucket: {
             username: envs.BITBUCKET_USERNAME,
@@ -438,9 +527,15 @@ const calculateValue = () => {
         bupt: {
             portal_cookie: envs.BUPT_PORTAL_COOKIE,
         },
+        caixin: {
+            cookie: envs.CAIXIN_COOKIE,
+        },
         civitai: {
             cookie: envs.CIVITAI_COOKIE,
         },
+        dianping: {
+            cookie: envs.DIANPING_COOKIE,
+        },
         dida365: {
             username: envs.DIDA365_USERNAME,
             password: envs.DIDA365_PASSWORD,
@@ -495,6 +590,9 @@ const calculateValue = () => {
         google: {
             fontsApiKey: envs.GOOGLE_FONTS_API_KEY,
         },
+        guozaoke: {
+            cookies: envs.GUOZAOKE_COOKIES,
+        },
         hefeng: {
             // weather
             key: envs.HEFENG_KEY,
@@ -506,7 +604,6 @@ const calculateValue = () => {
             username: envs.INITIUM_USERNAME,
             password: envs.INITIUM_PASSWORD,
             bearertoken: envs.INITIUM_BEARER_TOKEN,
-            iap_receipt: envs.INITIUM_IAP_RECEIPT,
         },
         instagram: {
             username: envs.IG_USERNAME,
@@ -521,12 +618,25 @@ const calculateValue = () => {
         javdb: {
             session: envs.JAVDB_SESSION,
         },
+        keylol: {
+            cookie: envs.KEYLOL_COOKIE,
+        },
         lastfm: {
             api_key: envs.LASTFM_API_KEY,
         },
         lightnovel: {
             cookie: envs.SECURITY_KEY,
         },
+        lorientlejour: {
+            token: envs.LORIENTLEJOUR_TOKEN,
+            username: envs.LORIENTLEJOUR_USERNAME,
+            password: envs.LORIENTLEJOUR_PASSWORD,
+        },
+        malaysiakini: {
+            email: envs.MALAYSIAKINI_EMAIL,
+            password: envs.MALAYSIAKINI_PASSWORD,
+            refreshToken: envs.MALAYSIAKINI_REFRESHTOKEN,
+        },
         manhuagui: {
             cookie: envs.MHGUI_COOKIE,
         },
@@ -566,6 +676,9 @@ const calculateValue = () => {
         notion: {
             key: envs.NOTION_TOKEN,
         },
+        patreon: {
+            sessionId: envs.PATREON_SESSION_ID,
+        },
         pianyuan: {
             cookie: envs.PIANYUAN_COOKIE,
         },
@@ -585,6 +698,9 @@ const calculateValue = () => {
         qingting: {
             id: envs.QINGTING_ID,
         },
+        readwise: {
+            accessToken: envs.READWISE_ACCESS_TOKEN,
+        },
         saraba1st: {
             cookie: envs.SARABA1ST_COOKIE,
         },
@@ -597,28 +713,51 @@ const calculateValue = () => {
         scihub: {
             host: envs.SCIHUB_HOST || 'https://sci-hub.se/',
         },
+        sis001: {
+            baseUrl: envs.SIS001_BASE_URL || 'https://sis001.com',
+        },
+        skeb: {
+            bearerToken: envs.SKEB_BEARER_TOKEN,
+        },
+        sorrycc: {
+            cookie: envs.SORRYCC_COOKIES,
+        },
         spotify: {
             clientId: envs.SPOTIFY_CLIENT_ID,
             clientSecret: envs.SPOTIFY_CLIENT_SECRET,
             refreshToken: envs.SPOTIFY_REFRESHTOKEN,
         },
+        sspai: {
+            bearertoken: envs.SSPAI_BEARERTOKEN,
+        },
         telegram: {
             token: envs.TELEGRAM_TOKEN,
             session: envs.TELEGRAM_SESSION,
             apiId: envs.TELEGRAM_API_ID,
             apiHash: envs.TELEGRAM_API_HASH,
             maxConcurrentDownloads: envs.TELEGRAM_MAX_CONCURRENT_DOWNLOADS,
+            proxy: {
+                host: envs.TELEGRAM_PROXY_HOST,
+                port: envs.TELEGRAM_PROXY_PORT,
+                secret: envs.TELEGRAM_PROXY_SECRET,
+            },
         },
         tophub: {
             cookie: envs.TOPHUB_COOKIE,
         },
+        tsdm39: {
+            cookie: envs.TSDM39_COOKIES,
+        },
         twitter: {
-            oauthTokens: envs.TWITTER_OAUTH_TOKEN?.split(','),
-            oauthTokenSecrets: envs.TWITTER_OAUTH_TOKEN_SECRET?.split(','),
-            username: envs.TWITTER_USERNAME,
-            password: envs.TWITTER_PASSWORD,
-            authenticationSecret: envs.TWITTER_AUTHENTICATION_SECRET,
-            cookie: envs.TWITTER_COOKIE,
+            username: envs.TWITTER_USERNAME?.split(','),
+            password: envs.TWITTER_PASSWORD?.split(','),
+            authenticationSecret: envs.TWITTER_AUTHENTICATION_SECRET?.split(','),
+            phoneOrEmail: envs.TWITTER_PHONE_OR_EMAIL?.split(','),
+            authToken: envs.TWITTER_AUTH_TOKEN?.split(','),
+        },
+        uestc: {
+            bbsCookie: envs.UESTC_BBS_COOKIE,
+            bbsAuthStr: envs.UESTC_BBS_AUTH_STR,
         },
         weibo: {
             app_key: envs.WEIBO_APP_KEY,
@@ -636,12 +775,23 @@ const calculateValue = () => {
             device_id: envs.XIAOYUZHOU_ID,
             refresh_token: envs.XIAOYUZHOU_TOKEN,
         },
+        xiaohongshu: {
+            cookie: envs.XIAOHONGSHU_COOKIE,
+        },
         ximalaya: {
             token: envs.XIMALAYA_TOKEN,
         },
+        xsijishe: {
+            cookie: envs.XSIJISHE_COOKIE,
+            user_agent: envs.XSIJISHE_USER_AGENT,
+        },
         xueqiu: {
             cookies: envs.XUEQIU_COOKIES,
         },
+        yamibo: {
+            salt: envs.YAMIBO_SALT,
+            auth: envs.YAMIBO_AUTH,
+        },
         youtube: {
             key: envs.YOUTUBE_KEY,
             clientId: envs.YOUTUBE_CLIENT_ID,

package/lib/errors/index.test.ts

@@ -22,13 +22,17 @@ describe('httperror', () => {
     it(`httperror`, async () => {
         const response = await request.get('/test/httperror');
         expect(response.status).toBe(503);
-        expect(response.text).toMatch('FetchError: [GET] "https://httpbingo.org/status/404": 404 Not Found');
+        expect(response.text).toContain('FetchError: [GET] "https://httpbingo.org/status/404": 404 Not Found');
     }, 20000);
 });
 
 describe('RequestInProgressError', () => {
-    it(`RequestInProgressError`, async () => {
+    it(`RequestInProgressError with retry`, async () => {
         const responses = await Promise.all([request.get('/test/slow'), request.get('/test/slow')]);
+        expect(new Set(responses.map((r) => r.status))).toEqual(new Set([200, 200]));
+    });
+    it(`RequestInProgressError`, async () => {
+        const responses = await Promise.all([request.get('/test/slow4'), request.get('/test/slow4')]);
         expect(new Set(responses.map((r) => r.status))).toEqual(new Set([200, 503]));
         expect(new Set(responses.map((r) => r.headers['cache-control']))).toEqual(new Set([`public, max-age=${config.cache.routeExpire}`, `public, max-age=${config.requestTimeout / 1000}`]));
         expect(responses.filter((r) => r.text.includes('RequestInProgressError: This path is currently fetching, please come back later!'))).toHaveLength(1);
@@ -63,19 +67,19 @@ describe('route throws an error', () => {
             const value = $(item).find('.debug-value').html()?.trim();
             switch (key) {
                 case 'Request Amount:':
-                    expect(value).toBe('9');
+                    expect(value).toBe('11');
                     break;
                 case 'Hot Routes:':
-                    expect(value).toBe('6 /test/:id/:params?<br>');
+                    expect(value).toBe('8 /test/:id/:params?<br>');
                     break;
                 case 'Hot Paths:':
-                    expect(value).toBe('2 /test/error<br>2 /test/slow<br>1 /test/httperror<br>1 /test/config-not-found-error<br>1 /test/invalid-parameter-error<br>1 /thisDoesNotExist<br>1 /<br>');
+                    expect(value).toBe('2 /test/error<br>2 /test/slow<br>2 /test/slow4<br>1 /test/httperror<br>1 /test/config-not-found-error<br>1 /test/invalid-parameter-error<br>1 /thisDoesNotExist<br>1 /<br>');
                     break;
                 case 'Hot Error Routes:':
                     expect(value).toBe('5 /test/:id/:params?<br>');
                     break;
                 case 'Hot Error Paths:':
-                    expect(value).toBe('2 /test/error<br>1 /test/httperror<br>1 /test/slow<br>1 /test/config-not-found-error<br>1 /test/invalid-parameter-error<br>1 /thisDoesNotExist<br>');
+                    expect(value).toBe('2 /test/error<br>1 /test/httperror<br>1 /test/slow4<br>1 /test/config-not-found-error<br>1 /test/invalid-parameter-error<br>1 /thisDoesNotExist<br>');
                     break;
                 default:
             }

package/lib/errors/index.tsx

@@ -1,20 +1,26 @@
 import { type NotFoundHandler, type ErrorHandler } from 'hono';
 import { getDebugInfo, setDebugInfo } from '@/utils/debug-info';
 import { config } from '@/config';
-import Sentry from '@sentry/node';
+import * as Sentry from '@sentry/node';
 import logger from '@/utils/logger';
 import Error from '@/views/error';
 
 import NotFoundError from './types/not-found';
 
+import { requestMetric } from '@/utils/otel';
+
 export const errorHandler: ErrorHandler = (error, ctx) => {
     const requestPath = ctx.req.path;
     const matchedRoute = ctx.req.routePath;
     const hasMatchedRoute = matchedRoute !== '/*';
 
     const debug = getDebugInfo();
-    if (ctx.res.headers.get('RSSHub-Cache-Status')) {
-        debug.hitCache++;
+    try {
+        if (ctx.res.headers.get('RSSHub-Cache-Status')) {
+            debug.hitCache++;
+        }
+    } catch {
+        // ignore
     }
     debug.error++;
 
@@ -61,8 +67,9 @@ export const errorHandler: ErrorHandler = (error, ctx) => {
     const message = `${error.name}: ${errorMessage}`;
 
     logger.error(`Error in ${requestPath}: ${message}`);
+    requestMetric.error({ path: matchedRoute, method: ctx.req.method, status: ctx.res.status });
 
-    return config.isPackage
+    return config.isPackage || ctx.req.query('format') === 'json'
         ? ctx.json({
               error: {
                   message: error.message ?? error,

package/lib/index.ts

@@ -10,12 +10,15 @@ const hostIPList = getLocalhostAddress();
 logger.info(`🎉 RSSHub is running on port ${port}! Cheers!`);
 logger.info('💖 Can you help keep this open source project alive? Please sponsor 👉 https://docs.rsshub.app/sponsor');
 logger.info(`🔗 Local: 👉 http://localhost:${port}`);
-for (const ip of hostIPList) {
-    logger.info(`🔗 Network: 👉 http://${ip}:${port}`);
+if (config.listenInaddrAny) {
+    for (const ip of hostIPList) {
+        logger.info(`🔗 Network: 👉 http://${ip}:${port}`);
+    }
 }
 
 const server = serve({
     fetch: app.fetch,
+    hostname: config.listenInaddrAny ? '::' : '127.0.0.1',
     port,
 });
 

package/lib/middleware/access-control.ts

@@ -3,12 +3,12 @@ import { config } from '@/config';
 import md5 from '@/utils/md5';
 import RejectError from '@/errors/types/reject';
 
-const reject = () => {
-    throw new RejectError('Authentication failed. Access denied.');
+const reject = (requestPath) => {
+    throw new RejectError(`Authentication failed. Access denied.\n${requestPath}`);
 };
 
 const middleware: MiddlewareHandler = async (ctx, next) => {
-    const requestPath = ctx.req.path;
+    const requestPath = new URL(ctx.req.url).pathname;
     const accessKey = ctx.req.query('key');
     const accessCode = ctx.req.query('code');
 
@@ -16,7 +16,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
         await next();
     } else {
         if (config.accessKey && !(config.accessKey === accessKey || accessCode === md5(requestPath + config.accessKey))) {
-            return reject();
+            return reject(requestPath);
         }
         await next();
     }