rsshub 1.0.0-master.f9e891b → 1.0.0-master.fa1dcc5

package/lib/utils/request-rewriter/fetch.ts

@@ -3,6 +3,7 @@ import { config } from '@/config';
 import undici, { Request, RequestInfo, RequestInit } from 'undici';
 import proxy from '@/utils/proxy';
 import { RateLimiterMemory, RateLimiterQueue } from 'rate-limiter-flexible';
+import { useRegisterRequest } from 'node-network-devtools';
 
 const limiter = new RateLimiterMemory({
     points: 10,
@@ -11,9 +12,19 @@ const limiter = new RateLimiterMemory({
 });
 
 const limiterQueue = new RateLimiterQueue(limiter, {
-    maxQueueSize: 500,
+    maxQueueSize: 4800,
 });
 
+export const useCustomHeader = (headers: Headers) => {
+    process.env.NODE_ENV === 'dev' &&
+        useRegisterRequest((req) => {
+            for (const [key, value] of headers.entries()) {
+                req.requestHeaders[key] = value;
+            }
+            return req;
+        });
+};
+
 const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: RequestInit) => {
     const request = new Request(input, init);
     const options: RequestInit = {};
@@ -46,6 +57,8 @@ const wrappedFetch: typeof undici.fetch = async (input: RequestInfo, init?: Requ
         request.headers.delete('x-prefer-proxy');
     }
 
+    config.enableRemoteDebugging && useCustomHeader(request.headers);
+
     // proxy
     if (!init?.dispatcher && proxy.dispatcher && (proxy.proxyObj.strategy !== 'on_retry' || isRetry)) {
         const proxyRegex = new RegExp(proxy.proxyObj.url_regex);

package/lib/views/error.tsx

@@ -26,7 +26,7 @@ const Index: FC<{
                     <p className="message">
                         Error Message:
                         <br />
-                        <code className="mt-2 block max-h-28 overflow-auto bg-zinc-100 align-bottom w-fit details">{message}</code>
+                        <code className="mt-2 block max-h-28 overflow-auto bg-zinc-100 align-bottom w-fit details whitespace-pre-line">{message}</code>
                     </p>
                     <p className="message">
                         Route: <code className="ml-2 bg-zinc-100">{errorRoute}</code>

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "rsshub",
-  "version": "1.0.0-master.f9e891b",
+  "version": "1.0.0-master.fa1dcc5",
   "description": "Make RSS Great Again!",
   "keywords": [
     "RSS"
@@ -32,24 +32,25 @@
     "*.yml": "eslint --cache --fix"
   },
   "dependencies": {
-    "@hono/node-server": "1.13.1",
-    "@hono/zod-openapi": "0.16.0",
+    "@bbob/html": "4.1.1",
+    "@bbob/preset-html5": "4.1.1",
+    "@hono/node-server": "1.13.7",
+    "@hono/zod-openapi": "0.18.0",
     "@notionhq/client": "2.2.15",
     "@opentelemetry/api": "1.9.0",
-    "@opentelemetry/exporter-prometheus": "0.53.0",
-    "@opentelemetry/exporter-trace-otlp-http": "0.53.0",
-    "@opentelemetry/resources": "1.26.0",
-    "@opentelemetry/sdk-metrics": "1.26.0",
-    "@opentelemetry/sdk-trace-base": "1.26.0",
-    "@opentelemetry/semantic-conventions": "1.27.0",
+    "@opentelemetry/exporter-prometheus": "0.55.0",
+    "@opentelemetry/exporter-trace-otlp-http": "0.55.0",
+    "@opentelemetry/resources": "1.28.0",
+    "@opentelemetry/sdk-metrics": "1.28.0",
+    "@opentelemetry/sdk-trace-base": "1.28.0",
+    "@opentelemetry/semantic-conventions": "1.28.0",
     "@postlight/parser": "2.2.3",
-    "@rss3/sdk": "0.0.17",
-    "@scalar/hono-api-reference": "0.5.147",
-    "@sentry/node": "7.116.0",
-    "@tonyrl/rand-user-agent": "2.0.79",
+    "@rss3/sdk": "0.0.23",
+    "@scalar/hono-api-reference": "0.5.161",
+    "@sentry/node": "7.119.1",
+    "@tonyrl/rand-user-agent": "2.0.81",
     "aes-js": "3.1.2",
     "art-template": "4.13.2",
-    "bbcodejs": "0.0.4",
     "cheerio": "1.0.0",
     "chrono-node": "2.7.7",
     "city-timezones": "1.3.0",
@@ -63,66 +64,69 @@
     "entities": "5.0.0",
     "etag": "1.8.1",
     "fanfou-sdk": "5.0.0",
-    "form-data": "4.0.0",
+    "form-data": "4.0.1",
     "googleapis": "144.0.0",
-    "hono": "4.6.3",
+    "hono": "4.6.12",
     "html-to-text": "9.0.5",
     "http-cookie-agent": "6.0.6",
     "https-proxy-agent": "7.0.5",
     "iconv-lite": "0.6.3",
-    "imapflow": "1.0.164",
+    "imapflow": "1.0.169",
     "instagram-private-api": "1.46.1",
     "ioredis": "5.4.1",
     "ip-regex": "5.0.0",
     "jsdom": "25.0.1",
     "json-bigint": "1.0.0",
     "jsrsasign": "10.9.0",
-    "lru-cache": "11.0.1",
+    "lru-cache": "11.0.2",
     "lz-string": "1.5.0",
     "mailparser": "3.7.1",
     "markdown-it": "14.1.0",
     "module-alias": "2.2.3",
+    "narou": "1.0.0",
     "notion-to-md": "3.1.1",
     "oauth-1.0a": "2.2.6",
-    "ofetch": "1.4.0",
+    "ofetch": "1.4.1",
     "otplib": "12.0.1",
     "pac-proxy-agent": "7.0.2",
-    "proxy-chain": "2.5.3",
+    "proxy-chain": "2.5.5",
     "puppeteer": "22.6.2",
     "puppeteer-extra": "3.3.6",
     "puppeteer-extra-plugin-stealth": "2.11.2",
     "puppeteer-extra-plugin-user-data-dir": "2.4.1",
     "puppeteer-extra-plugin-user-preferences": "2.4.1",
-    "query-string": "9.1.0",
-    "rate-limiter-flexible": "5.0.3",
-    "re2js": "0.4.1",
+    "query-string": "9.1.1",
+    "rate-limiter-flexible": "5.0.4",
+    "re2js": "0.4.3",
     "rfc4648": "1.5.3",
     "rss-parser": "3.13.0",
-    "sanitize-html": "2.13.0",
-    "simplecc-wasm": "1.0.0",
+    "sanitize-html": "2.13.1",
+    "simplecc-wasm": "1.1.0",
     "socks-proxy-agent": "8.0.4",
     "source-map": "0.7.4",
-    "telegram": "2.25.11",
+    "telegram": "2.26.8",
     "tiny-async-pool": "2.1.0",
-    "title": "3.5.3",
-    "tldts": "6.1.47",
+    "title": "4.0.1",
+    "tldts": "6.1.64",
     "tosource": "2.0.0-alpha.3",
-    "tough-cookie": "4.1.4",
-    "tsx": "4.19.1",
-    "twitter-api-v2": "1.17.2",
-    "undici": "6.19.8",
-    "uuid": "10.0.0",
-    "winston": "3.14.2",
-    "xxhash-wasm": "1.0.2",
+    "tough-cookie": "5.0.0",
+    "tsx": "4.19.2",
+    "twitter-api-v2": "1.18.2",
+    "ufo": "1.5.4",
+    "undici": "6.21.0",
+    "uuid": "11.0.3",
+    "winston": "3.17.0",
+    "xxhash-wasm": "1.1.0",
     "zod": "3.23.8"
   },
   "devDependencies": {
-    "@babel/preset-env": "7.25.4",
-    "@babel/preset-typescript": "7.24.7",
-    "@eslint/eslintrc": "3.1.0",
-    "@eslint/js": "9.11.1",
+    "@babel/preset-env": "7.26.0",
+    "@babel/preset-typescript": "7.26.0",
+    "@bbob/types": "4.1.1",
+    "@eslint/eslintrc": "3.2.0",
+    "@eslint/js": "9.15.0",
     "@microsoft/eslint-formatter-sarif": "3.1.0",
-    "@stylistic/eslint-plugin": "2.8.0",
+    "@stylistic/eslint-plugin": "2.11.0",
     "@types/aes-js": "3.1.4",
     "@types/babel__preset-env": "7.9.7",
     "@types/crypto-js": "4.2.2",
@@ -136,42 +140,42 @@
     "@types/json-bigint": "1.0.4",
     "@types/jsrsasign": "10.5.13",
     "@types/lint-staged": "13.3.0",
-    "@types/mailparser": "3.4.4",
+    "@types/mailparser": "3.4.5",
     "@types/markdown-it": "14.1.2",
     "@types/module-alias": "2.0.4",
-    "@types/node": "22.6.1",
+    "@types/node": "22.9.3",
     "@types/sanitize-html": "2.13.0",
     "@types/supertest": "6.0.2",
     "@types/tiny-async-pool": "2.0.3",
     "@types/title": "3.4.3",
-    "@types/tough-cookie": "4.0.5",
     "@types/uuid": "10.0.0",
-    "@typescript-eslint/eslint-plugin": "8.7.0",
-    "@typescript-eslint/parser": "8.7.0",
-    "@vercel/nft": "0.27.4",
+    "@typescript-eslint/eslint-plugin": "8.15.0",
+    "@typescript-eslint/parser": "8.15.0",
+    "@vercel/nft": "0.27.6",
     "@vitest/coverage-v8": "2.0.5",
-    "discord-api-types": "0.37.101",
-    "eslint": "9.11.1",
+    "discord-api-types": "0.37.107",
+    "eslint": "9.15.0",
     "eslint-config-prettier": "9.1.0",
     "eslint-nibble": "8.1.0",
-    "eslint-plugin-n": "17.10.3",
+    "eslint-plugin-n": "17.14.0",
     "eslint-plugin-prettier": "5.2.1",
-    "eslint-plugin-unicorn": "55.0.0",
-    "eslint-plugin-yml": "1.14.0",
+    "eslint-plugin-unicorn": "56.0.1",
+    "eslint-plugin-yml": "1.15.0",
     "fs-extra": "11.2.0",
-    "globals": "15.9.0",
-    "got": "14.4.2",
-    "husky": "9.1.6",
+    "globals": "15.12.0",
+    "got": "14.4.5",
+    "husky": "9.1.7",
     "js-beautify": "1.15.1",
     "lint-staged": "15.2.10",
     "mockdate": "3.0.5",
     "msw": "2.4.3",
+    "node-network-devtools": "1.0.22",
     "prettier": "3.3.3",
     "remark-parse": "11.0.0",
     "supertest": "7.0.0",
-    "typescript": "5.6.2",
+    "typescript": "5.6.3",
     "unified": "11.0.5",
-    "vite-tsconfig-paths": "5.0.1",
+    "vite-tsconfig-paths": "5.1.3",
     "vitest": "2.0.5",
     "yaml-eslint-parser": "1.2.3"
   },
@@ -181,7 +185,7 @@
   "scripts": {
     "build": "tsx scripts/workflow/build-routes.ts",
     "build:docs": "tsx scripts/workflow/build-docs.ts",
-    "dev": "cross-env NODE_ENV=dev tsx watch --clear-screen=false lib/index.ts",
+    "dev": "cross-env NODE_ENV=dev tsx watch --inspect --clear-screen=false lib/index.ts",
     "dev:cache": "cross-env NODE_ENV=production tsx watch --clear-screen=false lib/index.ts",
     "format": "prettier \"**/*.{ts,tsx,js,json}\" --write && eslint --cache --fix \"**/*.{ts,tsx,js,yml}\"",
     "format:check": "prettier \"**/*.{ts,tsx,js,json}\" --check && eslint --cache \"**/*.{ts,tsx,js,yml}\"",

package/lib/routes/78dm/templates/image.art

@@ -1 +0,0 @@
-<img src="{{ image }}">

package/lib/routes/a9vg/a9vg.ts

@@ -1,45 +0,0 @@
-import { Route } from '@/types';
-import got from '@/utils/got';
-import * as cheerio from 'cheerio';
-import { parseDate } from '@/utils/parse-date';
-import timezone from '@/utils/timezone';
-
-export const route: Route = {
-    path: '/',
-    radar: [
-        {
-            source: ['a9vg.com/list/news', 'a9vg.com/'],
-            target: '',
-        },
-    ],
-    name: 'Unknown',
-    maintainers: ['monnerHenster'],
-    handler,
-    url: 'a9vg.com/list/news',
-};
-
-async function handler() {
-    const baseUrl = 'http://www.a9vg.com';
-    const link = `${baseUrl}/list/news`;
-    const { data } = await got(link);
-
-    const $ = cheerio.load(data);
-    const list = $('.a9-rich-card-list li')
-        .toArray()
-        .map((elem) => {
-            const item = $(elem);
-            return {
-                title: item.find('.a9-rich-card-list_label').text(),
-                description: `<img src="${item.find('img').attr('src')}"><br>${item.find('.a9-rich-card-list_summary').text().trim()}`,
-                pubDate: timezone(parseDate(item.find('.a9-rich-card-list_infos').text().trim(), 'YYYY-MM-DD HH:mm:ss'), 8),
-                link: `${baseUrl}${item.find('.a9-rich-card-list_item').attr('href')}`,
-            };
-        });
-
-    return {
-        title: 'A9VG 电玩部落',
-        link,
-        description: $('meta[name="description"]').attr('content'),
-        item: list,
-    };
-}

package/lib/routes/bangumi/tv/other/followrank.ts

@@ -1,80 +0,0 @@
-import { Route } from '@/types';
-import { load } from 'cheerio';
-import { config } from '@/config';
-import ofetch from '@/utils/ofetch';
-
-export const route: Route = {
-    path: '/:type/followrank',
-    categories: ['anime'],
-    example: '/bangumi/anime/followrank',
-    parameters: { type: '类型：anime - 动画, book - 图书, music - 音乐, game - 游戏, real - 三次元' },
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
-    radar: [
-        {
-            source: ['bgm.tv/:type'],
-            target: '/:type/followrank',
-        },
-    ],
-    name: '成员关注榜',
-    maintainers: ['honue', 'zhoukuncheng'],
-    handler,
-};
-
-async function handler(ctx) {
-    let type = ctx.req.param('type');
-    if (!type || type === 'tv') {
-        type = 'anime';
-    }
-    const url = `https://bgm.tv/${type}`;
-
-    const response = await ofetch(url, {
-        headers: {
-            'User-Agent': config.trueUA,
-        },
-    });
-
-    const $ = load(response);
-
-    const items = [
-        ...$('#columnB > div:nth-child(4) > table > tbody')
-            .find('tr')
-            .toArray()
-            .map((item) => {
-                const aTag = $(item).children('td').next().find('a');
-                return {
-                    title: aTag.html(),
-                    link: 'https://bgm.tv' + aTag.attr('href'),
-                };
-            }),
-        ...$('#chl_subitem > ul')
-            .find('li')
-            .toArray()
-            .map((item) => ({
-                title: $(item).children('a').attr('title'),
-                link: 'https://bgm.tv' + $(item).children('a').attr('href'),
-            })),
-    ];
-
-    const RANK_TYPES = {
-        tv: '动画',
-        anime: '动画',
-        book: '图书',
-        music: '音乐',
-        game: '游戏',
-        real: '三次元',
-    };
-
-    return {
-        title: `BangumiTV 成员关注${RANK_TYPES[type]}榜`,
-        link: url,
-        item: items,
-        description: `BangumiTV 首页-成员关注${RANK_TYPES[type]}榜`,
-    };
-}

package/lib/routes/bangumi/tv/user/wish.ts

@@ -1,62 +0,0 @@
-import { Route } from '@/types';
-import got from '@/utils/got';
-import { load } from 'cheerio';
-import timezone from '@/utils/timezone';
-import { parseDate } from '@/utils/parse-date';
-import { config } from '@/config';
-export const route: Route = {
-    path: '/tv/user/wish/:id',
-    categories: ['anime'],
-    example: '/bangumi/tv/user/wish/sai',
-    parameters: { id: '用户 id, 在用户页面地址栏查看' },
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
-    radar: [
-        {
-            source: ['bgm.tv/anime/list/:id/wish'],
-        },
-    ],
-    name: '用户想看',
-    maintainers: ['honue'],
-    handler,
-};
-
-async function handler(ctx) {
-    const userid = ctx.req.param('id');
-    const url = `https://bgm.tv/anime/list/${userid}/wish`;
-    const response = await got({
-        url,
-        method: 'get',
-        headers: {
-            'User-Agent': config.trueUA,
-        },
-    });
-    const $ = load(response.body);
-
-    const username = $('.name').find('a').html();
-    const items = $('#browserItemList')
-        .find('li')
-        .toArray()
-        .map((item) => {
-            const aTag = $(item).find('h3').children('a');
-            const jdate = $(item).find('.collectInfo').find('span').html();
-            return {
-                title: aTag.html(),
-                link: 'https://bgm.tv' + aTag.attr('href'),
-                pubDate: timezone(parseDate(jdate), 0),
-            };
-        });
-
-    return {
-        title: `${username}想看的动画`,
-        link: url,
-        item: items,
-        description: `${username}想看的动画列表`,
-    };
-}

package/lib/routes/deeplearning/thebatch.ts

@@ -1,74 +0,0 @@
-import { Route } from '@/types';
-import cache from '@/utils/cache';
-import ofetch from '@/utils/ofetch';
-import * as cheerio from 'cheerio';
-import { parseDate } from '@/utils/parse-date';
-
-export const route: Route = {
-    path: '/thebatch',
-    categories: ['programming'],
-    example: '/deeplearning/thebatch',
-    parameters: {},
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
-    radar: [
-        {
-            source: ['www.deeplearning.ai/thebatch', 'www.deeplearning.ai/'],
-        },
-    ],
-    name: 'TheBatch 周报',
-    maintainers: ['nczitzk', 'juvenn'],
-    handler,
-    url: 'www.deeplearning.ai/thebatch',
-};
-
-async function handler() {
-    const baseUrl = 'https://www.deeplearning.ai';
-    const link = `${baseUrl}/the-batch/`;
-    const page = await ofetch(link);
-    const $ = cheerio.load(page);
-    const nextJs = $('script#__NEXT_DATA__').text();
-    const nextBuildId = JSON.parse(nextJs).buildId;
-
-    const listing = await ofetch(`${baseUrl}/_next/data/${nextBuildId}/the-batch.json`);
-
-    const items = listing.pageProps.posts.map((item) => ({
-        title: item.title,
-        link: `${link}${item.slug}`,
-        jsonUrl: `${baseUrl}/_next/data/${nextBuildId}/the-batch/${item.slug}.json`,
-        pubDate: parseDate(item.published_at),
-    }));
-
-    return {
-        title: 'The Batch - a new weekly newsletter from deeplearning.ai',
-        link,
-        item: await Promise.all(
-            items.map((item) =>
-                cache.tryGet(item.link, async () => {
-                    const resp = await ofetch(item.jsonUrl);
-                    const $ = cheerio.load(resp.pageProps.cmsData.post.html);
-
-                    $('a').each((_, ele) => {
-                        if (ele.attribs.href?.includes('utm_campaign')) {
-                            const url = new URL(ele.attribs.href);
-                            url.searchParams.delete('utm_campaign');
-                            url.searchParams.delete('utm_source');
-                            url.searchParams.delete('utm_medium');
-                            url.searchParams.delete('_hsenc');
-                            ele.attribs.href = url.href;
-                        }
-                    });
-
-                    item.description = $.html();
-                    return item;
-                })
-            )
-        ),
-    };
-}

package/lib/routes/gov/ndrc/zfxxgk/articles.ts

@@ -1,73 +0,0 @@
-import { Route } from '@/types';
-import cache from '@/utils/cache';
-import got from '@/utils/got';
-import { load } from 'cheerio';
-import { parseDate } from '@/utils/parse-date';
-
-export const route: Route = {
-    path: '/ndrc/zfxxgk/iteminfo',
-    categories: ['government'],
-    example: '/gov/ndrc/zfxxgk/iteminfo',
-    parameters: {},
-    features: {
-        requireConfig: false,
-        requirePuppeteer: false,
-        antiCrawler: false,
-        supportBT: false,
-        supportPodcast: false,
-        supportScihub: false,
-    },
-    name: '国家发展改革委 - 政府信息公开',
-    maintainers: ['howfool'],
-    handler,
-    url: 'zfxxgk.ndrc.gov.cn/web/dirlist.jsp',
-};
-
-async function handler() {
-    const homeUrl = 'https://zfxxgk.ndrc.gov.cn/web/dirlist.jsp';
-    const rootUrl = 'https://zfxxgk.ndrc.gov.cn/web/';
-
-    const response = await got({
-        method: 'get',
-        url: homeUrl,
-    });
-
-    const $ = load(response.data);
-
-    let items = $('div.zwgk-right .zwxxkg-result tr')
-        .toArray()
-        .slice(1)
-        .map((item) => {
-            item = $(item);
-
-            return {
-                title: item.find('a').text().replace(/^\s*/, ''),
-                link: rootUrl + item.find('a').attr('href'),
-                data: item.find('td:last').text(),
-            };
-        });
-
-    items = await Promise.all(
-        items.map((item) =>
-            cache.tryGet(item.link, async () => {
-                const detailResponse = await got({
-                    method: 'get',
-                    url: item.link,
-                });
-
-                const content = load(detailResponse.data);
-                content('[class$="top"]').remove();
-                item.description = content('.zwgkbg').html();
-                item.pubDate = parseDate(item.data);
-
-                return item;
-            })
-        )
-    );
-
-    return {
-        title: '国家发展改革委 - 政府信息公开',
-        link: homeUrl,
-        item: items,
-    };
-}