rsshub 1.0.0-master.ff3a9ce → 1.0.0-master.ff3ea5a

package/lib/routes/zju/sis/index.ts

@@ -0,0 +1,161 @@
+import { DataItem, Route } from '@/types';
+import cache from '@/utils/cache';
+import got from '@/utils/got';
+import { load } from 'cheerio';
+import { parseDate } from '@/utils/parse-date';
+
+const base = 'http://www.sis.zju.edu.cn/sischinese/';
+
+/**
+ * Map of category types to their corresponding IDs and titles
+ * Used to generate URLs and titles for different news categories
+ */
+const categoryMap = new Map([
+    [0, { id: '12614/list.htm', title: '浙江大学外国语学院-重要公告' }],
+    [1, { id: '12616/list.htm', title: '浙江大学外国语学院-最新通知' }],
+    [2, { id: '12617/list.htm', title: '浙江大学外国语学院-教育教学' }],
+    [3, { id: '12618/list.htm', title: '浙江大学外国语学院-科学研究' }],
+    [4, { id: '12619/list.htm', title: '浙江大学外国语学院-新闻动态' }],
+    [5, { id: '12620/list.htm', title: '浙江大学外国语学院-联系我们' }],
+    [6, { id: '12554/list.htm', title: '浙江大学外国语学院-党政管理' }],
+    [7, { id: '12563/list.htm', title: '浙江大学外国语学院-组织人事' }],
+    [8, { id: '12572/list.htm', title: '浙江大学外国语学院-科学研究' }],
+    [9, { id: '12577/list.htm', title: '浙江大学外国语学院-本科教育' }],
+    [10, { id: '12541/list.htm', title: '浙江大学外国语学院-研究生教育' }],
+    [11, { id: '12542/list.htm', title: '浙江大学外国语学院-学生思政' }],
+    [12, { id: 'xyll/list.htm', title: '浙江大学外国语学院-校友联络' }],
+    [13, { id: '12609/list.htm', title: '浙江大学外国语学院-对外交流' }],
+]);
+
+/**
+ * Fetches and parses news items from a specific category page
+ * @param categoryId - The category ID to fetch news from
+ * @returns Promise<DataItem[]> - Array of news items with basic information
+ */
+async function fetchNewsItemsByCategory(categoryId: string): Promise<DataItem[]> {
+    const response = await got({
+        method: 'get',
+        url: `${base}${categoryId}`,
+    });
+
+    const $ = load(response.data);
+    const newsItems = $('.news_list').find('li');
+
+    return newsItems.toArray().map((item) => {
+        const element = $(item);
+        const href = element.find('a').attr('href');
+        let title = element.find('a').attr('title');
+        if (!title) {
+            // If the title is not found, try to extract it from the link text
+            title = element.find('a').text().trim();
+        }
+
+        return {
+            title,
+            pubDate: parseDate(element.find('.news_meta').text()),
+            link: href ? new URL(href, base).href : undefined,
+        };
+    });
+}
+
+/**
+ * Enriches a news item with detailed content by fetching its full page
+ * @param item - The basic news item to enrich
+ * @param refererUrl - The referer URL to use when fetching the item details
+ * @returns Promise<DataItem> - The enriched news item with description, author, and full title
+ */
+async function enrichNewsItemWithDetails(item: DataItem, refererUrl: string): Promise<DataItem> {
+    // If the item doesn't have a link, return the item as-is
+    if (!item.link) {
+        return item;
+    }
+
+    return await cache.tryGet(item.link, async () => {
+        try {
+            // Some news items may link to pages that require ZJU private network access
+            // or university account authentication. We'll handle these gracefully.
+            const response = await got({
+                method: 'get',
+                url: item.link,
+                headers: {
+                    Referer: refererUrl,
+                },
+            });
+
+            const $ = load(response.data);
+
+            // Extract and set the full article content as description
+            const description = $('.wp_articlecontent').html();
+            if (description) {
+                item.description = description;
+            }
+
+            // Extract and clean the author information
+            let author = $('.arti_metas').find('.arti_publisher').text();
+            // Remove the '发布者：' (Publisher:) prefix and trim whitespace
+            author = author.replace('发布者：', '').trim();
+            if (author) {
+                item.author = author;
+            }
+
+            return item;
+        } catch {
+            // If there's an error accessing the page (network issues, authentication required, etc.),
+            // return the item with only the basic information we already have
+            return item;
+        }
+    });
+}
+
+export const route: Route = {
+    path: '/sis/:type',
+    categories: ['university'],
+    example: '/zju/sis/0',
+    parameters: { type: '分类，见下表' },
+    features: {
+        requireConfig: false,
+        requirePuppeteer: false,
+        antiCrawler: false,
+        supportBT: false,
+        supportPodcast: false,
+        supportScihub: false,
+    },
+    name: '外国语学院',
+    description: `| 重要公告 | 最新通知 | 教育教学 | 科学研究 | 新闻动态 | 联系我们 | 党政管理 | 组织人事 | 科学研究 | 本科教育 | 研究生教育 | 学生思政 | 校友联络 | 对外交流 |
+| -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- | -------- |
+| 0        | 1        | 2        | 3        | 4        | 5        | 6        | 7            | 8            | 9       | 10       | 11       | 12       | 13       |
+`,
+    maintainers: ['Alex222222222222'],
+    handler: handleSisRequest,
+    url: 'www.sis.zju.edu.cn',
+};
+
+/**
+ * Main handler function for processing SIS (School of International Studies) news requests
+ * @param ctx - The request context containing route parameters
+ * @returns Promise with RSS feed data including title, link, and news items
+ */
+async function handleSisRequest(ctx: { req: { param: (arg0: string) => string } }) {
+    const requestedType = Number.parseInt(ctx.req.param('type'));
+    const categoryInfo = categoryMap.get(requestedType);
+
+    // Validate the requested category type
+    if (!categoryInfo) {
+        const validTypes = [...categoryMap.keys()].join(', ');
+        throw new Error(`Invalid type: ${requestedType}. Valid types are: ${validTypes}`);
+    }
+
+    const categoryUrl = `${base}${categoryInfo.id}`;
+
+    // Fetch news items from all relevant categories
+    const allNewsItems = await fetchNewsItemsByCategory(categoryInfo.id);
+
+    // Enrich each news item with detailed content
+    const enrichedItems = await Promise.all(allNewsItems.map((item) => enrichNewsItemWithDetails(item, categoryUrl)));
+
+    return {
+        title: categoryInfo.title,
+        link: categoryUrl,
+        item: enrichedItems,
+    };
+}

package/lib/server.ts

@@ -0,0 +1,5 @@
+// This file is for compatibility with Vercel's deployment.
+
+import '@/utils/request-rewriter';
+
+export default (await import('./app-bootstrap')).default;

package/lib/types.ts

@@ -2,7 +2,7 @@ import type { Context } from 'hono';
 
 // Make sure it's synchronise with scripts/workflow/data.ts
 // and lib/routes/rsshub/routes.ts
-type Category =
+export type Category =
     | 'popular'
     | 'social-media'
     | 'new-media'
@@ -98,71 +98,89 @@ export type Data = {
     ttl?: number;
 };
 
-type Language =
+export type Language =
     | 'af'
-    | 'sq'
-    | 'eu'
-    | 'be'
-    | 'bg'
-    | 'ca'
-    | 'zh-CN'
-    | 'zh-TW'
-    | 'zh-HK'
-    | 'hr'
-    | 'cs'
     | 'ar-DZ'
-    | 'ar-SA'
-    | 'ar-MA'
     | 'ar-IQ'
     | 'ar-KW'
+    | 'ar-MA'
+    | 'ar-SA'
     | 'ar-TN'
+    | 'be'
+    | 'bg'
+    | 'ca'
+    | 'cs'
     | 'da'
-    | 'nl'
-    | 'nl-be'
-    | 'nl-nl'
+    | 'de'
+    | 'de-at'
+    | 'de-ch'
+    | 'de-de'
+    | 'de-li'
+    | 'de-lu'
+    | 'el'
     | 'en'
     | 'en-au'
     | 'en-bz'
     | 'en-ca'
+    | 'en-gb'
     | 'en-ie'
     | 'en-jm'
     | 'en-nz'
     | 'en-ph'
-    | 'en-za'
     | 'en-tt'
-    | 'en-gb'
     | 'en-us'
+    | 'en-za'
     | 'en-zw'
+    | 'es'
+    | 'es-ar'
+    | 'es-bo'
+    | 'es-cl'
+    | 'es-co'
+    | 'es-cr'
+    | 'es-do'
+    | 'es-ec'
+    | 'es-es'
+    | 'es-gt'
+    | 'es-hn'
+    | 'es-mx'
+    | 'es-ni'
+    | 'es-pa'
+    | 'es-pe'
+    | 'es-pr'
+    | 'es-py'
+    | 'es-sv'
+    | 'es-uy'
+    | 'es-ve'
     | 'et'
-    | 'fo'
+    | 'eu'
     | 'fi'
+    | 'fo'
     | 'fr'
     | 'fr-be'
     | 'fr-ca'
+    | 'fr-ch'
     | 'fr-fr'
     | 'fr-lu'
     | 'fr-mc'
-    | 'fr-ch'
-    | 'gl'
+    | 'ga'
     | 'gd'
-    | 'de'
-    | 'de-at'
-    | 'de-de'
-    | 'de-li'
-    | 'de-lu'
-    | 'de-ch'
-    | 'el'
+    | 'gl'
     | 'haw'
+    | 'hi'
+    | 'hr'
     | 'hu'
-    | 'is'
     | 'in'
-    | 'ga'
+    | 'is'
     | 'it'
-    | 'it-it'
     | 'it-ch'
+    | 'it-it'
     | 'ja'
     | 'ko'
     | 'mk'
+    | 'ne'
+    | 'nl'
+    | 'nl-be'
+    | 'nl-nl'
     | 'no'
     | 'pl'
     | 'pt'
@@ -174,35 +192,18 @@ type Language =
     | 'ru'
     | 'ru-mo'
     | 'ru-ru'
-    | 'sr'
     | 'sk'
     | 'sl'
-    | 'es'
-    | 'es-ar'
-    | 'es-bo'
-    | 'es-cl'
-    | 'es-co'
-    | 'es-cr'
-    | 'es-do'
-    | 'es-ec'
-    | 'es-sv'
-    | 'es-gt'
-    | 'es-hn'
-    | 'es-mx'
-    | 'es-ni'
-    | 'es-pa'
-    | 'es-py'
-    | 'es-pe'
-    | 'es-pr'
-    | 'es-es'
-    | 'es-uy'
-    | 'es-ve'
+    | 'sq'
+    | 'sr'
     | 'sv'
     | 'sv-fi'
     | 'sv-se'
     | 'tr'
     | 'uk'
-    | 'ne'
+    | 'zh-CN'
+    | 'zh-HK'
+    | 'zh-TW'
     | 'other';
 
 // namespace
@@ -346,6 +347,9 @@ interface RouteItem {
 
         /** Set to `true` if the feed supports Sci-Hub */
         supportScihub?: boolean;
+
+        /** Set to `true` if this feed is not safe for work */
+        nsfw?: boolean;
     };
 
     /**

package/lib/utils/helpers.ts

@@ -50,7 +50,7 @@ export function parseDuration(timeStr: string) {
     const clean = timeStr.trim().replaceAll(/[^\d:]/g, '');
     return clean
         .split(':')
-        .reverse()
+        .toReversed()
         .reduce((total, part, idx) => {
             const n = Number(part);
             if (Number.isNaN(n)) {

package/lib/utils/logger.ts

@@ -3,7 +3,7 @@ import winston from 'winston';
 import { config } from '@/config';
 
 let transports: (typeof winston.transports.File)[] = [];
-if (!config.noLogfiles) {
+if (!config.noLogfiles && !process.env.VERCEL) {
     transports = [
         new winston.transports.File({
             filename: path.resolve('logs/error.log'),

package/lib/utils/proxy/index.ts

@@ -3,20 +3,72 @@ import { PacProxyAgent } from 'pac-proxy-agent';
 import { HttpsProxyAgent } from 'https-proxy-agent';
 import { SocksProxyAgent } from 'socks-proxy-agent';
 import { ProxyAgent } from 'undici';
+import logger from '@/utils/logger';
 
 const proxyIsPAC = config.pacUri || config.pacScript;
 
 import pacProxy from './pac-proxy';
 import unifyProxy from './unify-proxy';
+import createMultiProxy, { type MultiProxyResult, type ProxyState } from './multi-proxy';
+
+interface ProxyExport {
+    agent: PacProxyAgent<string> | HttpsProxyAgent<string> | SocksProxyAgent | null;
+    dispatcher: ProxyAgent | null;
+    proxyUri?: string;
+    proxyObj: Record<string, any>;
+    proxyUrlHandler?: URL | null;
+    multiProxy?: MultiProxyResult;
+    getCurrentProxy: () => ProxyState | null;
+    markProxyFailed: (proxyUri: string) => void;
+    getAgentForProxy: (proxyState: ProxyState) => any;
+    getDispatcherForProxy: (proxyState: ProxyState) => ProxyAgent | null;
+}
 
 let proxyUri: string | undefined;
-let proxyObj: Record<string, any> | undefined;
+let proxyObj: Record<string, any> = {};
 let proxyUrlHandler: URL | null = null;
+let multiProxy: MultiProxyResult | undefined;
+
+const createAgentForProxy = (uri: string, proxyObj: Record<string, any>): any => {
+    if (uri.startsWith('http')) {
+        return new HttpsProxyAgent(uri, {
+            headers: {
+                'proxy-authorization': proxyObj?.auth ? `Basic ${proxyObj.auth}` : undefined,
+            },
+        });
+    } else if (uri.startsWith('socks')) {
+        return new SocksProxyAgent(uri);
+    }
+    return null;
+};
+
+const createDispatcherForProxy = (uri: string, proxyObj: Record<string, any>): ProxyAgent | null => {
+    if (uri.startsWith('http')) {
+        return new ProxyAgent({
+            uri,
+            token: proxyObj?.auth ? `Basic ${proxyObj.auth}` : undefined,
+            requestTls: {
+                rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0',
+            },
+        });
+    }
+    return null;
+};
+
 if (proxyIsPAC) {
     const proxy = pacProxy(config.pacUri, config.pacScript, config.proxy);
     proxyUri = proxy.proxyUri;
     proxyObj = proxy.proxyObj;
     proxyUrlHandler = proxy.proxyUrlHandler;
+} else if (config.proxyUris && config.proxyUris.length > 0) {
+    multiProxy = createMultiProxy(config.proxyUris, config.proxy);
+    proxyObj = multiProxy.proxyObj;
+    const currentProxy = multiProxy.getNextProxy();
+    if (currentProxy) {
+        proxyUri = currentProxy.uri;
+        proxyUrlHandler = currentProxy.urlHandler;
+    }
+    logger.info(`Multi-proxy initialized with ${config.proxyUris.length} proxies`);
 } else {
     const proxy = unifyProxy(config.proxyUri, config.proxy);
     proxyUri = proxy.proxyUri;
@@ -26,31 +78,63 @@ if (proxyIsPAC) {
 
 let agent: PacProxyAgent<string> | HttpsProxyAgent<string> | SocksProxyAgent | null = null;
 let dispatcher: ProxyAgent | null = null;
-if (proxyIsPAC) {
+
+if (proxyIsPAC && proxyUri) {
     agent = new PacProxyAgent(`pac+${proxyUri}`);
 } else if (proxyUri) {
-    if (proxyUri.startsWith('http')) {
-        agent = new HttpsProxyAgent(proxyUri, {
-            headers: {
-                'proxy-authorization': config.proxy?.auth ? `Basic ${config.proxy?.auth}` : undefined,
-            },
-        });
-        dispatcher = new ProxyAgent({
+    agent = createAgentForProxy(proxyUri, proxyObj);
+    dispatcher = createDispatcherForProxy(proxyUri, proxyObj);
+}
+
+const getCurrentProxy = (): ProxyState | null => {
+    if (multiProxy) {
+        return multiProxy.getNextProxy();
+    }
+    if (proxyUri) {
+        return {
             uri: proxyUri,
-            token: config.proxy?.auth ? `Basic ${config.proxy?.auth}` : undefined,
-            requestTls: {
-                rejectUnauthorized: process.env.NODE_TLS_REJECT_UNAUTHORIZED !== '0',
-            },
-        });
-    } else if (proxyUri.startsWith('socks')) {
-        agent = new SocksProxyAgent(proxyUri);
+            isActive: true,
+            failureCount: 0,
+            urlHandler: proxyUrlHandler,
+        };
     }
-}
+    return null;
+};
 
-export default {
+const markProxyFailed = (failedProxyUri: string) => {
+    if (multiProxy) {
+        multiProxy.markProxyFailed(failedProxyUri);
+        const nextProxy = multiProxy.getNextProxy();
+        if (nextProxy) {
+            proxyUri = nextProxy.uri;
+            proxyUrlHandler = nextProxy.urlHandler || null;
+            agent = createAgentForProxy(nextProxy.uri, proxyObj);
+            dispatcher = createDispatcherForProxy(nextProxy.uri, proxyObj);
+            logger.info(`Switched to proxy: ${nextProxy.uri}`);
+        } else {
+            logger.warn('No available proxies remaining');
+            agent = null;
+            dispatcher = null;
+            proxyUri = undefined;
+        }
+    }
+};
+
+const getAgentForProxy = (proxyState: ProxyState) => createAgentForProxy(proxyState.uri, proxyObj);
+
+const getDispatcherForProxy = (proxyState: ProxyState) => createDispatcherForProxy(proxyState.uri, proxyObj);
+
+const proxyExport: ProxyExport = {
     agent,
     dispatcher,
     proxyUri,
     proxyObj,
     proxyUrlHandler,
+    multiProxy,
+    getCurrentProxy,
+    markProxyFailed,
+    getAgentForProxy,
+    getDispatcherForProxy,
 };
+
+export default proxyExport;

package/lib/utils/proxy/multi-proxy.ts

@@ -0,0 +1,139 @@
+import { type Config } from '@/config';
+import logger from '@/utils/logger';
+import unifyProxy from './unify-proxy';
+
+export interface ProxyState {
+    uri: string;
+    isActive: boolean;
+    failureCount: number;
+    lastFailureTime?: number;
+    agent?: any;
+    dispatcher?: any;
+    urlHandler?: URL | null;
+}
+
+export interface MultiProxyResult {
+    currentProxy?: ProxyState | null;
+    allProxies: ProxyState[];
+    proxyObj: Config['proxy'];
+    getNextProxy: () => ProxyState | null;
+    markProxyFailed: (proxyUri: string) => void;
+    resetProxy: (proxyUri: string) => void;
+}
+
+const createMultiProxy = (proxyUris: string[], proxyObj: Config['proxy']): MultiProxyResult => {
+    const proxies: ProxyState[] = [];
+    let currentProxyIndex = 0;
+
+    for (const uri of proxyUris) {
+        const unifiedProxy = unifyProxy(uri, proxyObj);
+        if (unifiedProxy.proxyUri) {
+            proxies.push({
+                uri: unifiedProxy.proxyUri,
+                isActive: true,
+                failureCount: 0,
+                urlHandler: unifiedProxy.proxyUrlHandler,
+            });
+        }
+    }
+
+    if (proxies.length === 0) {
+        logger.warn('No valid proxies found in the provided list');
+        return {
+            allProxies: [],
+            proxyObj: proxyObj || {},
+            getNextProxy: () => null,
+            markProxyFailed: () => {},
+            resetProxy: () => {},
+        };
+    }
+
+    const healthCheckInterval = proxyObj?.healthCheckInterval || 60000;
+    const maxFailures = 3;
+
+    const healthCheck = () => {
+        const now = Date.now();
+        for (const proxy of proxies) {
+            if (!proxy.isActive && proxy.lastFailureTime && now - proxy.lastFailureTime > healthCheckInterval) {
+                proxy.isActive = true;
+                proxy.failureCount = 0;
+                delete proxy.lastFailureTime;
+                logger.info(`Proxy ${proxy.uri} marked as active again after health check`);
+            }
+        }
+    };
+
+    setInterval(healthCheck, healthCheckInterval);
+
+    const getNextProxy = (): ProxyState | null => {
+        const activeProxies = proxies.filter((p) => p.isActive);
+        if (activeProxies.length === 0) {
+            logger.warn('No active proxies available');
+            return null;
+        }
+
+        let nextProxy = activeProxies[currentProxyIndex % activeProxies.length];
+        let attempts = 0;
+
+        while (!nextProxy.isActive && attempts < activeProxies.length) {
+            currentProxyIndex = (currentProxyIndex + 1) % activeProxies.length;
+            nextProxy = activeProxies[currentProxyIndex];
+            attempts++;
+        }
+
+        if (!nextProxy.isActive) {
+            return null;
+        }
+
+        return nextProxy;
+    };
+
+    const markProxyFailed = (proxyUri: string) => {
+        const proxy = proxies.find((p) => p.uri === proxyUri);
+        if (proxy) {
+            proxy.failureCount++;
+            proxy.lastFailureTime = Date.now();
+            if (proxy.failureCount >= maxFailures) {
+                proxy.isActive = false;
+                logger.warn(`Proxy ${proxyUri} marked as inactive after ${maxFailures} failures`);
+            } else {
+                logger.warn(`Proxy ${proxyUri} failed (${proxy.failureCount}/${maxFailures})`);
+            }
+
+            const activeProxies = proxies.filter((p) => p.isActive);
+            if (activeProxies.length > 0) {
+                currentProxyIndex = (currentProxyIndex + 1) % activeProxies.length;
+                const nextProxy = getNextProxy();
+                if (nextProxy) {
+                    logger.info(`Switching to proxy: ${nextProxy.uri}`);
+                }
+            }
+        }
+    };
+
+    const resetProxy = (proxyUri: string) => {
+        const proxy = proxies.find((p) => p.uri === proxyUri);
+        if (proxy) {
+            proxy.isActive = true;
+            proxy.failureCount = 0;
+            delete proxy.lastFailureTime;
+            logger.info(`Proxy ${proxyUri} manually reset`);
+        }
+    };
+
+    const currentProxy = getNextProxy();
+    if (currentProxy) {
+        logger.info(`Initial proxy selected: ${currentProxy.uri}`);
+    }
+
+    return {
+        currentProxy,
+        allProxies: proxies,
+        proxyObj: proxyObj || {},
+        getNextProxy,
+        markProxyFailed,
+        resetProxy,
+    };
+};
+
+export default createMultiProxy;

package/lib/utils/proxy/unify-proxy.ts

@@ -4,7 +4,7 @@ import logger from '@/utils/logger';
 const defaultProtocol = 'http';
 const possibleProtocol = ['http', 'https', 'socks', 'socks4', 'socks4a', 'socks5', 'socks5h'];
 
-const unifyProxy = (proxyUri: Config['proxyUri'], proxyObj: Config['proxy']) => {
+const unifyProxy = (proxyUri: Config['proxyUri'] | string, proxyObj: Config['proxy']) => {
     proxyObj = proxyObj || {};
     const [oriProxyUri, oriProxyObj] = [proxyUri, proxyObj];
     proxyObj = { ...proxyObj };
@@ -110,4 +110,6 @@ const unifyProxy = (proxyUri: Config['proxyUri'], proxyObj: Config['proxy']) =>
     return { proxyUri, proxyObj, proxyUrlHandler };
 };
 
+export const unifyProxies = (proxyUris: string[], proxyObj: Config['proxy']) => proxyUris.map((uri) => unifyProxy(uri, proxyObj)).filter((result) => result.proxyUri);
+
 export default unifyProxy;

package/lib/utils/puppeteer-utils.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it, vi, afterEach } from 'vitest';
 import { parseCookieArray, constructCookieArray, setCookies, getCookies } from '@/utils/puppeteer-utils';
 import puppeteer from '@/utils/puppeteer';
-import type { Browser } from 'puppeteer';
+import type { Browser } from 'rebrowser-puppeteer';
 
 let browser: Browser | null = null;