rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa1dcc5

package/lib/config.ts

@@ -5,12 +5,14 @@ import { ofetch } from 'ofetch';
 let envs = process.env;
 
 export type Config = {
+    // app config
     disallowRobot: boolean;
     enableCluster?: string;
     isPackage: boolean;
     nodeName?: string;
     puppeteerWSEndpoint?: string;
     chromiumExecutablePath?: string;
+    // network
     connect: {
         port: number;
     };
@@ -20,6 +22,7 @@ export type Config = {
     ua: string;
     trueUA: string;
     allowOrigin?: string;
+    // cache
     cache: {
         type: string;
         requestTimeout: number;
@@ -32,6 +35,7 @@ export type Config = {
     redis: {
         url: string;
     };
+    // proxy
     proxyUri?: string;
     proxy: {
         protocol?: string;
@@ -43,7 +47,9 @@ export type Config = {
     };
     pacUri?: string;
     pacScript?: string;
+    // access control
     accessKey?: string;
+    // logging
     debugInfo: string;
     loggerLevel: string;
     noLogfiles?: boolean;
@@ -56,6 +62,8 @@ export type Config = {
         dsn?: string;
         routeTimeout: number;
     };
+    enableRemoteDebugging?: boolean;
+    // feed config
     hotlink: {
         template?: string;
         includePaths?: string[];
@@ -78,6 +86,8 @@ export type Config = {
         promptTitle: string;
         promptDescription: string;
     };
+
+    // Route-specific Configurations
     bilibili: {
         cookies: Record<string, string | undefined>;
         dmImgList?: string;
@@ -94,6 +104,9 @@ export type Config = {
     bupt: {
         portal_cookie?: string;
     };
+    caixin: {
+        cookie?: string;
+    };
     civitai: {
         cookie?: string;
     };
@@ -239,6 +252,9 @@ export type Config = {
     notion: {
         key?: string;
     };
+    patreon: {
+        sessionId?: string;
+    };
     pianyuan: {
         cookie?: string;
     };
@@ -293,6 +309,14 @@ export type Config = {
     telegram: {
         token?: string;
         session?: string;
+        apiId?: number;
+        apiHash?: string;
+        maxConcurrentDownloads?: number;
+        proxy?: {
+            host?: string;
+            port?: number;
+            secret?: string;
+        };
     };
     tophub: {
         cookie?: string;
@@ -327,6 +351,9 @@ export type Config = {
         device_id?: string;
         refresh_token?: string;
     };
+    xiaohongshu: {
+        cookie?: string;
+    };
     ximalaya: {
         token?: string;
     };
@@ -415,7 +442,6 @@ const calculateValue = () => {
         requestTimeout: toInt(envs.REQUEST_TIMEOUT, 30000), // Milliseconds to wait for the server to end the response before aborting the request
         ua: envs.UA ?? (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' })),
         trueUA: TRUE_UA,
-        // cors request
         allowOrigin: envs.ALLOW_ORIGIN,
         // cache
         cache: {
@@ -459,6 +485,7 @@ const calculateValue = () => {
             dsn: envs.SENTRY,
             routeTimeout: toInt(envs.SENTRY_ROUTE_TIMEOUT, 30000),
         },
+        enableRemoteDebugging: toBoolean(envs.ENABLE_REMOTE_DEBUGGING, false),
         // feed config
         hotlink: {
             template: envs.HOTLINK_TEMPLATE,
@@ -500,6 +527,9 @@ const calculateValue = () => {
         bupt: {
             portal_cookie: envs.BUPT_PORTAL_COOKIE,
         },
+        caixin: {
+            cookie: envs.CAIXIN_COOKIE,
+        },
         civitai: {
             cookie: envs.CIVITAI_COOKIE,
         },
@@ -646,6 +676,9 @@ const calculateValue = () => {
         notion: {
             key: envs.NOTION_TOKEN,
         },
+        patreon: {
+            sessionId: envs.PATREON_SESSION_ID,
+        },
         pianyuan: {
             cookie: envs.PIANYUAN_COOKIE,
         },
@@ -703,6 +736,11 @@ const calculateValue = () => {
             apiId: envs.TELEGRAM_API_ID,
             apiHash: envs.TELEGRAM_API_HASH,
             maxConcurrentDownloads: envs.TELEGRAM_MAX_CONCURRENT_DOWNLOADS,
+            proxy: {
+                host: envs.TELEGRAM_PROXY_HOST,
+                port: envs.TELEGRAM_PROXY_PORT,
+                secret: envs.TELEGRAM_PROXY_SECRET,
+            },
         },
         tophub: {
             cookie: envs.TOPHUB_COOKIE,
@@ -737,6 +775,9 @@ const calculateValue = () => {
             device_id: envs.XIAOYUZHOU_ID,
             refresh_token: envs.XIAOYUZHOU_TOKEN,
         },
+        xiaohongshu: {
+            cookie: envs.XIAOHONGSHU_COOKIE,
+        },
         ximalaya: {
             token: envs.XIMALAYA_TOKEN,
         },

package/lib/middleware/access-control.ts

@@ -3,8 +3,8 @@ import { config } from '@/config';
 import md5 from '@/utils/md5';
 import RejectError from '@/errors/types/reject';
 
-const reject = () => {
-    throw new RejectError('Authentication failed. Access denied.');
+const reject = (requestPath) => {
+    throw new RejectError(`Authentication failed. Access denied.\n${requestPath}`);
 };
 
 const middleware: MiddlewareHandler = async (ctx, next) => {
@@ -16,7 +16,7 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
         await next();
     } else {
         if (config.accessKey && !(config.accessKey === accessKey || accessCode === md5(requestPath + config.accessKey))) {
-            return reject();
+            return reject(requestPath);
         }
         await next();
     }

package/lib/middleware/cache.ts

@@ -8,7 +8,7 @@ import { Data } from '@/types';
 
 const bypassList = new Set(['/', '/robots.txt', '/logo.png', '/favicon.ico']);
 // only give cache string, as the `!` condition tricky
-// md5 is used to shrink key size
+// XXH64 is used to shrink key size
 // plz, write these tips in comments!
 const middleware: MiddlewareHandler = async (ctx, next) => {
     if (!cacheModule.status.available || bypassList.has(ctx.req.path)) {
@@ -16,9 +16,11 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
         return;
     }
 
+    const requestPath = ctx.req.path;
+    const limit = ctx.req.query('limit') ? `:${ctx.req.query('limit')}` : '';
     const { h64ToString } = await xxhash();
-    const key = 'rsshub:koa-redis-cache:' + h64ToString(ctx.req.path);
-    const controlKey = 'rsshub:path-requested:' + h64ToString(ctx.req.path);
+    const key = 'rsshub:koa-redis-cache:' + h64ToString(requestPath + limit);
+    const controlKey = 'rsshub:path-requested:' + h64ToString(requestPath + limit);
 
     const isRequesting = await cacheModule.globalCache.get(controlKey);
 

package/lib/middleware/template.test.ts

@@ -108,4 +108,10 @@ describe('template', () => {
         expect(parsed.items[0].enclosure?.length).toBe('3661');
         expect(parsed.items[0].itunes.duration).toBe('10:10:10');
     });
+
+    it(`redirect`, async () => {
+        const response = await app.request('/test/redirect');
+        expect(response.status).toBe(301);
+        expect(response.headers.get('location')).toBe('/test/1');
+    });
 });

package/lib/middleware/template.tsx

@@ -102,7 +102,9 @@ const middleware: MiddlewareHandler = async (ctx, next) => {
         return ctx.json(result);
     }
 
-    if (ctx.get('no-content')) {
+    if (ctx.get('redirect')) {
+        return ctx.redirect(ctx.get('redirect'), 301);
+    } else if (ctx.get('no-content')) {
         return ctx.body(null);
     } else {
         // retain .ums for backward compatibility

package/lib/router.js

@@ -874,9 +874,6 @@ router.get('/guet/xwzx/:type?', lazyloadRouteHandler('./routes/guet/news'));
 // はてな匿名ダイアリー
 router.get('/hatena/anonymous_diary/archive', lazyloadRouteHandler('./routes/hatena/anonymous_diary/archive'));
 
-// IEEE Xplore [Sci Journal]
-router.get('/ieee/author/:aid/:sortType/:count?', lazyloadRouteHandler('./routes/ieee/author'));
-
 // PNAS [Sci Journal]
 // router.get('/pnas/:topic?', lazyloadRouteHandler('./routes/pnas/index'));
 
@@ -1455,9 +1452,6 @@ router.get('/deepl/blog/:lang?', lazyloadRouteHandler('./routes/deepl/blog'));
 router.get('/muchong/journal/:type?', lazyloadRouteHandler('./routes/muchong/journal'));
 router.get('/muchong/:id/:type?/:sort?', lazyloadRouteHandler('./routes/muchong/index'));
 
-// 求是网
-router.get('/qstheory/:category?', lazyloadRouteHandler('./routes/qstheory/index'));
-
 // 生命时报
 router.get('/lifetimes/:category?', lazyloadRouteHandler('./routes/lifetimes/index'));
 

package/lib/routes/141jav/index.ts

@@ -66,7 +66,7 @@ async function handler(ctx) {
     const $ = load(response.data);
 
     if (getSubPath(ctx) === '/') {
-        ctx.redirect(`/141jav${$('.overview').first().attr('href')}`);
+        ctx.set('redirect', `/141jav${$('.overview').first().attr('href')}`);
         return;
     }
 

package/lib/routes/141ppv/index.ts

@@ -66,7 +66,7 @@ async function handler(ctx) {
     const $ = load(response.data);
 
     if (getSubPath(ctx) === '/') {
-        ctx.redirect(`/141ppv${$('.overview').first().attr('href')}`);
+        ctx.set('redirect', `/141ppv${$('.overview').first().attr('href')}`);
         return;
     }
 

package/lib/routes/36kr/hot-list.ts

@@ -26,7 +26,7 @@ const categories = {
 
 export const route: Route = {
     path: '/hot-list/:category?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/36kr/hot-list',
     parameters: { category: '分类，默认为24小时热榜' },
     features: {

package/lib/routes/36kr/index.ts

@@ -18,7 +18,7 @@ const shortcuts = {
 
 export const route: Route = {
     path: '/:category/:subCategory?/:keyword?',
-    categories: ['new-media'],
+    categories: ['new-media', 'popular'],
     example: '/36kr/newsflashes',
     parameters: {
         category: '分类，必填项',