rsshub 1.0.0-master.f9ec483 → 1.0.0-master.fa1dcc5

package/lib/utils/request-rewriter/fetch.test.ts

@@ -0,0 +1,94 @@
+import { getCurrentCell, setCurrentCell } from 'node-network-devtools';
+import { useCustomHeader } from './fetch';
+import { describe, beforeEach, afterEach, test, expect } from 'vitest';
+
+const getInitRequest = () =>
+    ({
+        requestHeaders: {} as Record<string, string>,
+        id: '',
+        loadCallFrames: () => {},
+        cookies: '',
+        requestData: '',
+        responseData: '',
+        responseHeaders: {},
+        responseInfo: {},
+    }) satisfies NonNullable<ReturnType<typeof getCurrentCell>>['request'];
+
+enum Env {
+    dev = 'dev',
+    production = 'production',
+    test = 'test',
+}
+
+describe('useCustomHeader', () => {
+    let originalEnv: string;
+
+    beforeEach(() => {
+        originalEnv = process.env.NODE_ENV || Env.test;
+        process.env.ENABLE_REMOTE_DEBUGGING = 'true';
+    });
+
+    afterEach(() => {
+        process.env.NODE_ENV = originalEnv;
+    });
+
+    test('should register request with custom headers in dev environment', () => {
+        process.env.NODE_ENV = Env.dev;
+
+        const headers = new Headers();
+        const headerText = 'authorization';
+        const headerValue = 'Bearer token';
+        headers.set(headerText, headerValue);
+
+        const req = getInitRequest();
+        setCurrentCell({
+            request: req,
+            pipes: [],
+            isAborted: false,
+        });
+
+        useCustomHeader(headers);
+
+        const cell = getCurrentCell();
+        expect(cell).toBeDefined();
+
+        let request = req;
+        if (cell) {
+            for (const { pipe } of cell.pipes) {
+                request = pipe(request);
+            }
+        }
+
+        expect(request.requestHeaders[headerText]).toEqual(headerValue);
+    });
+
+    test('should not register request in non-dev environment', () => {
+        process.env.NODE_ENV = Env.production;
+
+        const headers = new Headers();
+        const headerText = 'content-type';
+        const headerValue = 'application/json';
+
+        headers.set(headerText, headerValue);
+        const req = getInitRequest();
+
+        setCurrentCell({
+            request: req,
+            pipes: [],
+            isAborted: false,
+        });
+        useCustomHeader(headers);
+
+        const cell = getCurrentCell();
+        expect(cell).toBeDefined();
+
+        let request = req;
+        if (cell) {
+            for (const { pipe } of cell.pipes) {
+                request = pipe(request);
+            }
+        }
+
+        expect(req.requestHeaders[headerText]).toBeUndefined();
+    });
+});

package/lib/utils/request-rewriter/fetch.ts

@@ -3,6 +3,7 @@ import { config } from '@/config';
 import undici, { Request, RequestInfo, RequestInit } from 'undici';
 import proxy from '@/utils/proxy';
 import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
+import { useRegisterRequest } from 'node-network-devtools';
 
 const limiter = new RateLimiterMemory({
     points: 10,
@@ -11,9 +12,19 @@ const limiter = new RateLimiterMemory({
 });
 
 const limiterQueue = new RateLimiterQueue(limiter, {
-    maxQueueSize: 5000,
+    maxQueueSize: 4800,
 });
 
+export const useCustomHeader = (headers: Headers) => {
+    process.env.NODE_ENV === 'dev' &&
+        useRegisterRequest((req) => {
+            for (const [key, value] of headers.entries()) {
+                req.requestHeaders[key] = value;
+            }
+            return req;
+        });
+};
+
 const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: RequestInit) => {
     const request = new Request(input, init);
     const options: RequestInit = {};
@@ -46,6 +57,8 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ
         request.headers.delete('x-prefer-proxy');
     }
 
+    config.enableRemoteDebugging && useCustomHeader(request.headers);
+
     // proxy
     if (!init?.dispatcher && proxy.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
         const proxyRegex = new RegExp(proxy.proxyObj.url_regex);

package/lib/views/error.tsx

@@ -26,7 +26,7 @@ const Index: FC<{
                     <p className="message">
                         Error Message:
                         <br />
-                        <code className="mt-2 block max-h-28 overflow-auto bg-zinc-100 align-bottom w-fit details">{message}</code>
+                        <code className="mt-2 block max-h-28 overflow-auto bg-zinc-100 align-bottom w-fit details whitespace-pre-line">{message}</code>
                     </p>
                     <p className="message">
                         Route: <code className="ml-2 bg-zinc-100">{errorRoute}</code>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rsshub",
-  "version": "1.0.0-master.f9ec483",
+  "version": "1.0.0-master.fa1dcc5",
   "description": "Make RSS Great Again!",
   "keywords": [
     "RSS"
@@ -34,19 +34,19 @@
   "dependencies": {
     "@bbob/html": "4.1.1",
     "@bbob/preset-html5": "4.1.1",
-    "@hono/node-server": "1.13.3",
-    "@hono/zod-openapi": "0.16.4",
+    "@hono/node-server": "1.13.7",
+    "@hono/zod-openapi": "0.18.0",
     "@notionhq/client": "2.2.15",
     "@opentelemetry/api": "1.9.0",
-    "@opentelemetry/exporter-prometheus": "0.54.0",
-    "@opentelemetry/exporter-trace-otlp-http": "0.54.0",
-    "@opentelemetry/resources": "1.27.0",
-    "@opentelemetry/sdk-metrics": "1.27.0",
-    "@opentelemetry/sdk-trace-base": "1.27.0",
-    "@opentelemetry/semantic-conventions": "1.27.0",
+    "@opentelemetry/exporter-prometheus": "0.55.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.55.0",
+    "@opentelemetry/resources": "1.28.0",
+    "@opentelemetry/sdk-metrics": "1.28.0",
+    "@opentelemetry/sdk-trace-base": "1.28.0",
+    "@opentelemetry/semantic-conventions": "1.28.0",
     "@postlight/parser": "2.2.3",
     "@rss3/sdk": "0.0.23",
-    "@scalar/hono-api-reference": "0.5.158",
+    "@scalar/hono-api-reference": "0.5.161",
     "@sentry/node": "7.119.1",
     "@tonyrl/rand-user-agent": "2.0.81",
     "aes-js": "3.1.2",
@@ -66,12 +66,12 @@
     "fanfou-sdk": "5.0.0",
     "form-data": "4.0.1",
     "googleapis": "144.0.0",
-    "hono": "4.6.8",
+    "hono": "4.6.12",
     "html-to-text": "9.0.5",
     "http-cookie-agent": "6.0.6",
     "https-proxy-agent": "7.0.5",
     "iconv-lite": "0.6.3",
-    "imapflow": "1.0.165",
+    "imapflow": "1.0.169",
     "instagram-private-api": "1.46.1",
     "ioredis": "5.4.1",
     "ip-regex": "5.0.0",
@@ -83,12 +83,13 @@
     "mailparser": "3.7.1",
     "markdown-it": "14.1.0",
     "module-alias": "2.2.3",
+    "narou": "1.0.0",
     "notion-to-md": "3.1.1",
     "oauth-1.0a": "2.2.6",
     "ofetch": "1.4.1",
     "otplib": "12.0.1",
     "pac-proxy-agent": "7.0.2",
-    "proxy-chain": "2.5.4",
+    "proxy-chain": "2.5.5",
     "puppeteer": "22.6.2",
     "puppeteer-extra": "3.3.6",
     "puppeteer-extra-plugin-stealth": "2.11.2",
@@ -96,35 +97,36 @@
     "puppeteer-extra-plugin-user-preferences": "2.4.1",
     "query-string": "9.1.1",
     "rate-limiter-flexible": "5.0.4",
-    "re2js": "0.4.2",
+    "re2js": "0.4.3",
     "rfc4648": "1.5.3",
     "rss-parser": "3.13.0",
     "sanitize-html": "2.13.1",
     "simplecc-wasm": "1.1.0",
     "socks-proxy-agent": "8.0.4",
     "source-map": "0.7.4",
-    "telegram": "2.26.6",
+    "telegram": "2.26.8",
     "tiny-async-pool": "2.1.0",
-    "title": "3.5.3",
-    "tldts": "6.1.57",
+    "title": "4.0.1",
+    "tldts": "6.1.64",
     "tosource": "2.0.0-alpha.3",
     "tough-cookie": "5.0.0",
     "tsx": "4.19.2",
-    "twitter-api-v2": "1.18.1",
-    "undici": "6.20.1",
-    "uuid": "11.0.2",
-    "winston": "3.15.0",
-    "xxhash-wasm": "1.0.2",
+    "twitter-api-v2": "1.18.2",
+    "ufo": "1.5.4",
+    "undici": "6.21.0",
+    "uuid": "11.0.3",
+    "winston": "3.17.0",
+    "xxhash-wasm": "1.1.0",
     "zod": "3.23.8"
   },
   "devDependencies": {
     "@babel/preset-env": "7.26.0",
     "@babel/preset-typescript": "7.26.0",
     "@bbob/types": "4.1.1",
-    "@eslint/eslintrc": "3.1.0",
-    "@eslint/js": "9.13.0",
+    "@eslint/eslintrc": "3.2.0",
+    "@eslint/js": "9.15.0",
     "@microsoft/eslint-formatter-sarif": "3.1.0",
-    "@stylistic/eslint-plugin": "2.10.1",
+    "@stylistic/eslint-plugin": "2.11.0",
     "@types/aes-js": "3.1.4",
     "@types/babel__preset-env": "7.9.7",
     "@types/crypto-js": "4.2.2",
@@ -141,38 +143,39 @@
     "@types/mailparser": "3.4.5",
     "@types/markdown-it": "14.1.2",
     "@types/module-alias": "2.0.4",
-    "@types/node": "22.8.6",
+    "@types/node": "22.9.3",
     "@types/sanitize-html": "2.13.0",
     "@types/supertest": "6.0.2",
     "@types/tiny-async-pool": "2.0.3",
     "@types/title": "3.4.3",
     "@types/uuid": "10.0.0",
-    "@typescript-eslint/eslint-plugin": "8.12.2",
-    "@typescript-eslint/parser": "8.12.2",
-    "@vercel/nft": "0.27.5",
+    "@typescript-eslint/eslint-plugin": "8.15.0",
+    "@typescript-eslint/parser": "8.15.0",
+    "@vercel/nft": "0.27.6",
     "@vitest/coverage-v8": "2.0.5",
-    "discord-api-types": "0.37.103",
-    "eslint": "9.13.0",
+    "discord-api-types": "0.37.107",
+    "eslint": "9.15.0",
     "eslint-config-prettier": "9.1.0",
     "eslint-nibble": "8.1.0",
-    "eslint-plugin-n": "17.12.0",
+    "eslint-plugin-n": "17.14.0",
     "eslint-plugin-prettier": "5.2.1",
-    "eslint-plugin-unicorn": "56.0.0",
+    "eslint-plugin-unicorn": "56.0.1",
     "eslint-plugin-yml": "1.15.0",
     "fs-extra": "11.2.0",
-    "globals": "15.11.0",
-    "got": "14.4.3",
-    "husky": "9.1.6",
+    "globals": "15.12.0",
+    "got": "14.4.5",
+    "husky": "9.1.7",
     "js-beautify": "1.15.1",
     "lint-staged": "15.2.10",
     "mockdate": "3.0.5",
     "msw": "2.4.3",
+    "node-network-devtools": "1.0.22",
     "prettier": "3.3.3",
     "remark-parse": "11.0.0",
     "supertest": "7.0.0",
     "typescript": "5.6.3",
     "unified": "11.0.5",
-    "vite-tsconfig-paths": "5.0.1",
+    "vite-tsconfig-paths": "5.1.3",
     "vitest": "2.0.5",
     "yaml-eslint-parser": "1.2.3"
   },

package/lib/routes/78dm/templates/image.art

@@ -1 +0,0 @@
-<img src="{{ image }}">

package/lib/routes/deeplearning/thebatch.ts

@@ -1,74 +0,0 @@
-import { Route } from '@/types';
-import cache from '@/utils/cache';
-import ofetch from '@/utils/ofetch';
-import * as cheerio from 'cheerio';
-import { parseDate } from '@/utils/parse-date';
-
-export const route: Route = {
-    path: '/thebatch',
-    categories: ['programming'],
-    example: '/deeplearning/thebatch',
-    parameters: {},
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
-    radar: [
-        {
-            source: ['www.deeplearning.ai/thebatch', 'www.deeplearning.ai/'],
-        },
-    ],
-    name: 'TheBatch 周报',
-    maintainers: ['nczitzk', 'juvenn'],
-    handler,
-    url: 'www.deeplearning.ai/thebatch',
-};
-
-async function handler() {
-    const baseUrl = 'https://www.deeplearning.ai';
-    const link = `${baseUrl}/the-batch/`;
-    const page = await ofetch(link);
-    const $ = cheerio.load(page);
-    const nextJs = $('script#__NEXT_DATA__').text();
-    const nextBuildId = JSON.parse(nextJs).buildId;
-
-    const listing = await ofetch(`${baseUrl}/_next/data/${nextBuildId}/the-batch.json`);
-
-    const items = listing.pageProps.posts.map((item) => ({
-        title: item.title,
-        link: `${link}${item.slug}`,
-        jsonUrl: `${baseUrl}/_next/data/${nextBuildId}/the-batch/${item.slug}.json`,
-        pubDate: parseDate(item.published_at),
-    }));
-
-    return {
-        title: 'The Batch - a new weekly newsletter from deeplearning.ai',
-        link,
-        item: await Promise.all(
-            items.map((item) =>
-                cache.tryGet(item.link, async () => {
-                    const resp = await ofetch(item.jsonUrl);
-                    const $ = cheerio.load(resp.pageProps.cmsData.post.html);
-
-                    $('a').each((_, ele) => {
-                        if (ele.attribs.href?.includes('utm_campaign')) {
-                            const url = new URL(ele.attribs.href);
-                            url.searchParams.delete('utm_campaign');
-                            url.searchParams.delete('utm_source');
-                            url.searchParams.delete('utm_medium');
-                            url.searchParams.delete('_hsenc');
-                            ele.attribs.href = url.href;
-                        }
-                    });
-
-                    item.description = $.html();
-                    return item;
-                })
-            )
-        ),
-    };
-}

package/lib/routes/syosetu/chapter.ts

@@ -1,87 +0,0 @@
-import { Route } from '@/types';
-import cache from '@/utils/cache';
-import got from '@/utils/got';
-import { load } from 'cheerio';
-import timezone from '@/utils/timezone';
-import { parseDate } from '@/utils/parse-date';
-import { CookieJar } from 'tough-cookie';
-
-const cookieJar = new CookieJar();
-cookieJar.setCookieSync('over18=yes', 'https://novel18.syosetu.com/');
-
-export const route: Route = {
-    path: '/chapter/:id',
-    categories: ['reading'],
-    example: '/syosetu/chapter/n1976ey',
-    parameters: { id: 'Novel id, can be found in URL' },
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
-    radar: [
-        {
-            source: ['novel18.syosetu.com/:id'],
-        },
-    ],
-    name: 'chapter',
-    maintainers: ['huangliangshusheng'],
-    handler,
-    description: `Eg: \`https://ncode.syosetu.com/n1976ey/\``,
-};
-
-async function handler(ctx) {
-    const id = ctx.req.param('id');
-    const limit = Number.parseInt(ctx.req.query('limit')) || 5;
-    const link = `https://ncode.syosetu.com/${id}`;
-    const $ = load(await get(link));
-
-    const title = $('p.novel_title').text();
-    const description = $('#novel_ex').html();
-
-    const chapter_list = $('dl.novel_sublist2')
-        .toArray()
-        .map((chapter) => {
-            const $_chapter = $(chapter);
-            const chapter_link = $_chapter.find('a');
-            return {
-                title: chapter_link.text(),
-                link: chapter_link.attr('href'),
-                pubDate: timezone(parseDate($_chapter.find('dt').text(), 'YYYY/MM/DD HH:mm'), +9),
-            };
-        })
-        .sort((a, b) => (a.pubDate <= b.pubDate ? 1 : -1))
-        .slice(0, limit);
-
-    const item_list = await Promise.all(
-        chapter_list.map((chapter) =>
-            cache.tryGet(chapter.link, async () => {
-                chapter.link = `https://ncode.syosetu.com${chapter.link}`;
-                const content = load(await get(chapter.link));
-                chapter.description = content('#novel_honbun').html();
-                return chapter;
-            })
-        )
-    );
-
-    return {
-        title,
-        description,
-        link,
-        language: 'ja',
-        item: item_list,
-    };
-}
-
-const get = async (url) => {
-    const response = await got({
-        method: 'get',
-        url,
-        cookieJar,
-    });
-
-    return response.data;
-};

package/lib/routes/xiaohongshu/notes.ts

@@ -1,39 +0,0 @@
-import { Route } from '@/types';
-import cache from '@/utils/cache';
-import { getNotes, formatText, formatNote } from './util';
-
-export const route: Route = {
-    path: '/user/:user_id/notes/fulltext',
-    radar: [
-        {
-            source: ['xiaohongshu.com/user/profile/:user_id'],
-            target: '/user/:user_id/notes',
-        },
-    ],
-    name: '用户笔记 全文',
-    maintainers: ['howerhe'],
-    handler,
-    example: '/xiaohongshu/user/52d8c541b4c4d60e6c867480/notes/fulltext',
-    features: {
-        antiCrawler: true,
-        requirePuppeteer: true,
-    },
-    parameters: {
-        user_id: 'user id, length 24 characters',
-    },
-};
-
-async function handler(ctx) {
-    const userId = ctx.req.param('user_id');
-    const url = `https://www.xiaohongshu.com/user/profile/${userId}`;
-
-    const { user, notes } = await getNotes(url, cache);
-
-    return {
-        title: `${user.nickname} - 笔记 • 小红书 / RED`,
-        description: formatText(user.desc),
-        image: user.imageb || user.images,
-        link: url,
-        item: notes.map((item) => formatNote(url, item)),
-    };
-}

package/lib/routes-deprecated/furaffinity/browse.js

@@ -1,44 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    // 传入参数
-    const nsfw = String(ctx.params.nsfw);
-
-    // 判断传入的参数nsfw
-    let url = 'https://faexport.spangle.org.uk/browse.json?sfw=1';
-    if (nsfw === '1') {
-        url = 'https://faexport.spangle.org.uk/browse.json';
-    }
-    // 发起 HTTP GET 请求
-    const response = await got({
-        method: 'get',
-        url,
-        headers: {
-            Referer: `https://faexport.spangle.org.uk/`,
-        },
-    });
-
-    const data = response.data;
-
-    ctx.state.data = {
-        // 源标题
-        title: `Fur Affinity Browse`,
-        // 源链接
-        link: `https://www.furaffinity.net/browse/`,
-        // 源说明
-        description: `Fur Affinity Browse`,
-
-        // 遍历此前获取的数据
-        item: data.map((item) => ({
-            // 标题
-            title: item.title,
-            // 正文
-            description: `<img src="${item.thumbnail}">`,
-            // 链接
-            link: item.link,
-            // 作者
-            author: item.name,
-            // 由于源API未提供日期，故无pubDate
-        })),
-    };
-};

package/lib/routes-deprecated/furaffinity/commissions.js

@@ -1,39 +0,0 @@
-const got = require('@/utils/got');
-
-module.exports = async (ctx) => {
-    // 传入参数
-    const username = String(ctx.params.username);
-
-    // 发起 HTTP GET 请求
-    const response = await got({
-        method: 'get',
-        url: `https://faexport.spangle.org.uk/user/${username}/commissions.json`,
-        headers: {
-            Referer: `https://faexport.spangle.org.uk/`,
-        },
-    });
-
-    const data = response.data;
-
-    ctx.state.data = {
-        // 源标题
-        title: `${username}'s Commissions`,
-        // 源链接
-        link: `https://www.furaffinity.net/commissions/${username}/`,
-        // 源说明
-        description: `Fur Affinity ${username}'s Commissions`,
-
-        // 遍历此前获取的数据
-        item: data.map((item) => ({
-            // 标题
-            title: item.title,
-            // 正文
-            description: `${item.description} <br> <img src="${item.submission.thumbnail}"> `,
-            // 链接
-            link: item.submission.link,
-            // 作者
-            author: username,
-            // 由于源API未提供日期，故无pubDate
-        })),
-    };
-};

package/lib/routes-deprecated/furaffinity/favorites.js

@@ -1,56 +0,0 @@
-const got = require('@/utils/got');
-const cheerio = require('cheerio');
-
-module.exports = async (ctx) => {
-    // 传入参数
-    const nsfw = String(ctx.params.nsfw);
-    const username = String(ctx.params.username);
-
-    // 添加参数username以及判断传入的参数nsfw
-    let url = `https://faexport.spangle.org.uk/user/${username}/favorites.rss?sfw=1`;
-    if (nsfw === '1') {
-        url = `https://faexport.spangle.org.uk/user/${username}/favorites.rss`;
-    }
-
-    // 发起 HTTP GET 请求
-    const response = await got({
-        method: 'get',
-        url,
-        headers: {
-            Referer: `https://faexport.spangle.org.uk/`,
-        },
-    });
-
-    // 使用 cheerio 加载返回的 HTML
-
-    const data = response.data;
-    const $ = cheerio.load(data, {
-        xmlMode: true,
-    });
-
-    const list = $('item');
-
-    ctx.state.data = {
-        // 源标题
-        title: `${username}'s Favorites`,
-        // 源链接
-        link: `https://www.furaffinity.net/favorites/${username}/`,
-        // 源说明
-        description: `Fur Affinity ${username}'s Favorites`,
-
-        // 遍历此前获取的数据
-        item:
-            list &&
-            list
-                .map((index, item) => {
-                    item = $(item);
-                    return {
-                        title: item.find('title').text(),
-                        description: item.find('description').text(),
-                        link: item.find('link').text(),
-                        pubDate: new Date(item.find('pubDate').text()).toUTCString(),
-                    };
-                })
-                .get(),
-    };
-};