smart-image-scraper-mcp 2.1.1 → 2.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smart-image-scraper-mcp",
-  "version": "2.1.1",
+  "version": "2.3.0",
   "description": "全网智能图片抓取 MCP 服务器 - 支持 Bing/Google 图片搜索、验证和下载",
   "main": "src/index.js",
   "type": "module",

package/src/infrastructure/cache.js

@@ -26,11 +26,23 @@ export class LRUCache {
   }
 
   /**
-   * 生成缓存键
+   * 生成缓存键 - 优化版本
+   * 使用简单字符串拼接代替 JSON.stringify
    */
   _generateKey(key) {
     if (typeof key === 'string') return key;
-    return JSON.stringify(key);
+    if (typeof key === 'object' && key !== null) {
+      // 快速生成键：只使用关键字段
+      const parts = [];
+      for (const k of Object.keys(key).sort()) {
+        const v = key[k];
+        if (v !== undefined && v !== null && v !== '') {
+          parts.push(`${k}:${v}`);
+        }
+      }
+      return parts.join('|');
+    }
+    return String(key);
   }
 
   /**

package/src/infrastructure/httpClient.js

@@ -1,19 +1,42 @@
 /**
- * HTTP 客户端封装
- * 配置通用的 User-Agent 和 Timeout
+ * HTTP 客户端封装 - 高性能版本
+ * 配置连接池复用、Keep-Alive、超时控制
  */
 
 import axios from 'axios';
+import http from 'http';
+import https from 'https';
 import config from '../config/index.js';
 import logger from './logger.js';
 
+// HTTP 连接池配置 - 复用 TCP 连接，大幅提升性能
+const httpAgent = new http.Agent({
+  keepAlive: true,           // 启用 Keep-Alive
+  maxSockets: 50,            // 最大并发连接数
+  maxFreeSockets: 10,        // 最大空闲连接数
+  timeout: 30000,            // 连接超时 30 秒
+  scheduling: 'fifo',        // 先进先出调度
+});
+
+const httpsAgent = new https.Agent({
+  keepAlive: true,
+  maxSockets: 50,
+  maxFreeSockets: 10,
+  timeout: 30000,
+  scheduling: 'fifo',
+  rejectUnauthorized: false, // 允许自签名证书
+});
+
 const httpClient = axios.create({
   timeout: config.REQUEST_TIMEOUT,
+  httpAgent,                 // ✅ 使用连接池
+  httpsAgent,                // ✅ 使用连接池
   headers: {
     'User-Agent': config.USER_AGENT,
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
     'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
     'Accept-Encoding': 'gzip, deflate, br',
+    'Connection': 'keep-alive', // ✅ 保持连接
   },
   maxRedirects: 5,
   validateStatus: (status) => status < 500,

package/src/infrastructure/requestQueue.js

@@ -0,0 +1,291 @@
+/**
+ * 请求队列管理器 - 多线程队列机制
+ * 管理并发请求，确保资源正确释放
+ */
+
+import logger from './logger.js';
+import { metrics } from './metrics.js';
+
+/**
+ * 请求队列管理器
+ */
+export class RequestQueue {
+  constructor(options = {}) {
+    this.maxConcurrent = options.maxConcurrent || 5;  // 最大并发数
+    this.maxQueueSize = options.maxQueueSize || 20;   // 最大队列长度
+    this.requestTimeout = options.requestTimeout || 60000; // 请求超时 60 秒
+    
+    this.queue = [];           // 等待队列
+    this.active = new Map();   // 活跃请求 Map<requestId, requestInfo>
+    this.completed = [];       // 已完成请求（保留最近 100 个）
+    this.maxCompleted = 100;
+    
+    this.stats = {
+      totalProcessed: 0,
+      totalSuccess: 0,
+      totalFailed: 0,
+      totalTimeout: 0,
+    };
+    
+    // 定期清理超时请求
+    this.cleanupInterval = setInterval(() => this._cleanupTimeouts(), 5000);
+  }
+
+  /**
+   * 生成请求 ID
+   */
+  generateRequestId() {
+    return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  /**
+   * 提交请求到队列
+   * @param {Function} task - 异步任务函数
+   * @param {Object} metadata - 请求元数据
+   * @returns {Promise<Object>} - 任务结果
+   */
+  async submit(task, metadata = {}) {
+    const requestId = this.generateRequestId();
+    
+    // 检查队列是否已满
+    if (this.queue.length >= this.maxQueueSize) {
+      logger.warn(`[Queue] Request rejected: queue full (${this.queue.length}/${this.maxQueueSize})`);
+      metrics.recordError('queue_full');
+      return {
+        success: false,
+        error: `队列已满，请稍后重试。当前队列: ${this.queue.length}`,
+        requestId,
+        queueStatus: this.getStatus(),
+      };
+    }
+
+    // 创建请求包装
+    const requestInfo = {
+      id: requestId,
+      metadata,
+      createdAt: Date.now(),
+      status: 'queued',
+      resolve: null,
+      reject: null,
+    };
+
+    // 创建 Promise
+    const promise = new Promise((resolve, reject) => {
+      requestInfo.resolve = resolve;
+      requestInfo.reject = reject;
+      requestInfo.task = task;
+    });
+
+    // 加入队列
+    this.queue.push(requestInfo);
+    logger.info(`[Queue] Request queued: ${requestId} (queue: ${this.queue.length}, active: ${this.active.size})`);
+    
+    // 尝试处理队列
+    this._processQueue();
+
+    // 设置超时（修复内存泄漏：保存 timeoutId 以便清理）
+    let timeoutId;
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutId = setTimeout(() => {
+        reject(new Error('Request timeout'));
+      }, this.requestTimeout);
+    });
+
+    try {
+      const result = await Promise.race([promise, timeoutPromise]);
+      clearTimeout(timeoutId); // ✅ 清理超时定时器
+      return result;
+    } catch (error) {
+      clearTimeout(timeoutId); // ✅ 清理超时定时器
+      if (error.message === 'Request timeout') {
+        this.stats.totalTimeout++;
+        this._removeFromActive(requestId);
+        this._removeFromQueue(requestId); // ✅ 也从队列中移除
+        return {
+          success: false,
+          error: '请求超时',
+          requestId,
+        };
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * 从队列中移除请求
+   */
+  _removeFromQueue(requestId) {
+    const index = this.queue.findIndex(r => r.id === requestId);
+    if (index !== -1) {
+      this.queue.splice(index, 1);
+    }
+  }
+
+  /**
+   * 处理队列中的请求
+   */
+  _processQueue() {
+    while (this.queue.length > 0 && this.active.size < this.maxConcurrent) {
+      const requestInfo = this.queue.shift();
+      this._executeRequest(requestInfo);
+    }
+  }
+
+  /**
+   * 执行单个请求
+   */
+  async _executeRequest(requestInfo) {
+    const { id, task, resolve, metadata } = requestInfo;
+    
+    // 移入活跃列表
+    requestInfo.status = 'processing';
+    requestInfo.startedAt = Date.now();
+    this.active.set(id, requestInfo);
+    
+    logger.info(`[Queue] Processing: ${id} (active: ${this.active.size})`);
+
+    try {
+      // 执行任务
+      const result = await task();
+      
+      // 标记完成
+      requestInfo.status = 'completed';
+      requestInfo.completedAt = Date.now();
+      requestInfo.duration = requestInfo.completedAt - requestInfo.startedAt;
+      
+      this.stats.totalProcessed++;
+      this.stats.totalSuccess++;
+      
+      // 添加请求信息到结果
+      result.requestId = id;
+      result.processingTime = requestInfo.duration;
+      
+      resolve(result);
+      
+      logger.info(`[Queue] Completed: ${id} in ${requestInfo.duration}ms`);
+    } catch (error) {
+      requestInfo.status = 'failed';
+      requestInfo.error = error.message;
+      
+      this.stats.totalProcessed++;
+      this.stats.totalFailed++;
+      
+      resolve({
+        success: false,
+        error: error.message,
+        requestId: id,
+      });
+      
+      logger.error(`[Queue] Failed: ${id} - ${error.message}`);
+    } finally {
+      // 从活跃列表移除
+      this._removeFromActive(id);
+      
+      // 保存到已完成列表
+      this._addToCompleted(requestInfo);
+      
+      // 继续处理队列
+      this._processQueue();
+    }
+  }
+
+  /**
+   * 从活跃列表移除
+   */
+  _removeFromActive(requestId) {
+    this.active.delete(requestId);
+  }
+
+  /**
+   * 添加到已完成列表
+   */
+  _addToCompleted(requestInfo) {
+    this.completed.push({
+      id: requestInfo.id,
+      status: requestInfo.status,
+      duration: requestInfo.duration,
+      completedAt: requestInfo.completedAt,
+    });
+    
+    // 限制已完成列表大小
+    if (this.completed.length > this.maxCompleted) {
+      this.completed.shift();
+    }
+  }
+
+  /**
+   * 清理超时请求（仅作为备份机制，主超时由 Promise.race 处理）
+   */
+  _cleanupTimeouts() {
+    const now = Date.now();
+    // 使用更长的超时时间作为备份清理（比主超时多 10 秒）
+    const backupTimeout = this.requestTimeout + 10000;
+    
+    for (const [id, info] of this.active.entries()) {
+      if (now - info.startedAt > backupTimeout) {
+        logger.warn(`[Queue] Backup timeout cleanup: ${id}`);
+        // 仅从活跃列表移除，不再调用 reject（避免重复处理）
+        this._removeFromActive(id);
+        this.stats.totalTimeout++;
+      }
+    }
+    
+    // 清理队列中等待过久的请求
+    const queueTimeout = 30000; // 队列等待超过 30 秒
+    this.queue = this.queue.filter(info => {
+      if (now - info.createdAt > queueTimeout) {
+        logger.warn(`[Queue] Queue timeout: ${info.id}`);
+        info.resolve({
+          success: false,
+          error: '队列等待超时',
+          requestId: info.id,
+        });
+        return false;
+      }
+      return true;
+    });
+  }
+
+  /**
+   * 获取队列状态
+   */
+  getStatus() {
+    return {
+      queueLength: this.queue.length,
+      activeCount: this.active.size,
+      maxConcurrent: this.maxConcurrent,
+      maxQueueSize: this.maxQueueSize,
+      stats: { ...this.stats },
+      activeRequests: Array.from(this.active.entries()).map(([id, info]) => ({
+        id,
+        status: info.status,
+        duration: Date.now() - info.startedAt,
+        metadata: info.metadata,
+      })),
+    };
+  }
+
+  /**
+   * 清理资源
+   */
+  cleanup() {
+    clearInterval(this.cleanupInterval);
+    
+    // 拒绝所有等待中的请求
+    for (const info of this.queue) {
+      info.reject(new Error('Queue shutdown'));
+    }
+    this.queue = [];
+    
+    logger.info('[Queue] Cleanup completed');
+  }
+}
+
+// 全局请求队列实例
+export const requestQueue = new RequestQueue({
+  maxConcurrent: 5,
+  maxQueueSize: 20,
+  requestTimeout: 60000,
+});
+
+export default requestQueue;

package/src/providers/bingScraper.js

@@ -16,7 +16,7 @@ export class BingScraper extends BaseScraper {
   }
 
   /**
-   * 搜索 Bing 图片
+   * 搜索 Bing 图片 - 并行翻页版本
    * @param {string} keyword - 搜索关键词
    * @param {number} limit - 需要获取的图片数量
    * @param {Object} options - 搜索选项
@@ -24,63 +24,55 @@ export class BingScraper extends BaseScraper {
    */
   async search(keyword, limit = 10, options = {}) {
     this.options = options;
-    const urls = new Set();
-    let offset = 0;
-    const pageSize = 35; // Bing 每页大约35张图片
+    const pageSize = 35;
+    
+    // 计算需要的页数，并行获取
+    const pagesNeeded = Math.min(Math.ceil(limit / pageSize), 5); // 最多 5 页
+    const offsets = Array.from({ length: pagesNeeded }, (_, i) => i * pageSize);
+    
+    logger.info(`[Bing] Searching "${keyword}" - ${pagesNeeded} pages in parallel`);
 
     try {
-      while (urls.size < limit) {
-        const searchUrl = this._buildSearchUrl(keyword, offset);
-        logger.info(`Searching Bing Images: ${keyword}, offset: ${offset}`);
-        
-        const response = await withRetry(
-          () => httpClient.get(searchUrl, { timeout: 15000 }), // 添加超时
-          {
-            maxRetries: 2, // 减少重试次数
-            retryCondition: isRetryableError,
-          }
-        );
-        
-        if (response.status !== 200) {
-          logger.warn(`Bing search failed with status: ${response.status}`);
-          break;
-        }
-        
-        // 极速模式：最小延迟
-        await this._delay(50);
-
-        const newUrls = this._parseResponse(response.data);
-        
-        if (newUrls.length === 0) {
-          logger.info(`No more images found for: ${keyword}`);
-          break;
-        }
-
-        newUrls.forEach(url => {
-          if (urls.size < limit) {
-            urls.add(url);
-          }
-        });
-
-        // 如果已经获取足够数量，直接跳出
-        if (urls.size >= limit) {
-          break;
-        }
-
-        offset += pageSize;
-        
-        // 防止无限循环
-        if (offset > 200) { // 减少最大偏移量
-          logger.warn('Reached maximum offset limit');
-          break;
+      // 并行获取所有页面
+      const pagePromises = offsets.map(offset => this._fetchPage(keyword, offset));
+      const results = await Promise.allSettled(pagePromises);
+      
+      // 合并结果
+      const urls = new Set();
+      for (const result of results) {
+        if (result.status === 'fulfilled') {
+          result.value.forEach(url => {
+            if (urls.size < limit) {
+              urls.add(url);
+            }
+          });
         }
       }
+
+      logger.info(`[Bing] Complete: ${urls.size} URLs for "${keyword}"`);
+      return Array.from(urls);
     } catch (error) {
       logger.error(`Bing search error for "${keyword}"`, { message: error.message });
+      return [];
     }
+  }
 
-    logger.info(`Bing search complete: found ${urls.size} URLs for "${keyword}"`);
-    return Array.from(urls);
+  /**
+   * 获取单页结果
+   */
+  async _fetchPage(keyword, offset) {
+    const searchUrl = this._buildSearchUrl(keyword, offset);
+    
+    const response = await withRetry(
+      () => httpClient.get(searchUrl, { timeout: 10000 }),
+      { maxRetries: 1, retryCondition: isRetryableError }
+    );
+    
+    if (response.status !== 200) {
+      return [];
+    }
+    
+    return this._parseResponse(response.data);
   }
 
   /**

package/src/providers/googleScraper.js

@@ -24,65 +24,63 @@ export class GoogleScraper extends BaseScraper {
    */
   async search(keyword, limit = 10, options = {}) {
     this.options = options;
-    const urls = new Set();
-    let start = 0;
     const pageSize = 20;
+    
+    // 计算需要的页数，并行获取
+    const pagesNeeded = Math.min(Math.ceil(limit / pageSize), 5); // 最多 5 页
+    const starts = Array.from({ length: pagesNeeded }, (_, i) => i * pageSize);
+    
+    logger.info(`[Google] Searching "${keyword}" - ${pagesNeeded} pages in parallel`);
 
     try {
-      while (urls.size < limit && start < 100) { // 减少最大偏移
-        const searchUrl = this._buildSearchUrl(keyword, start);
-        logger.info(`Searching Google Images: ${keyword}, start: ${start}`);
-        
-        const response = await withRetry(
-          () => httpClient.get(searchUrl, {
-            timeout: 15000, // 添加超时
-            headers: {
-              'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-              'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-              'Accept-Language': 'en-US,en;q=0.5',
-              'Referer': 'https://www.google.com/',
-            },
-          }),
-          {
-            maxRetries: 2, // 减少重试次数
-            retryCondition: isRetryableError,
-          }
-        );
-        
-        if (response.status !== 200) {
-          logger.warn(`Google search failed with status: ${response.status}`);
-          break;
-        }
-
-        const newUrls = this._parseResponse(response.data);
-        
-        if (newUrls.length === 0) {
-          logger.info(`No more images found for: ${keyword}`);
-          break;
-        }
-
-        newUrls.forEach(url => {
-          if (urls.size < limit) {
-            urls.add(url);
-          }
-        });
-
-        // 如果已经获取足够数量，直接跳出
-        if (urls.size >= limit) {
-          break;
+      // 并行获取所有页面
+      const pagePromises = starts.map(start => this._fetchPage(keyword, start));
+      const results = await Promise.allSettled(pagePromises);
+      
+      // 合并结果
+      const urls = new Set();
+      for (const result of results) {
+        if (result.status === 'fulfilled') {
+          result.value.forEach(url => {
+            if (urls.size < limit) {
+              urls.add(url);
+            }
+          });
         }
-
-        start += pageSize;
-        
-        // 极速模式：最小延迟
-        await this._delay(50);
       }
+
+      logger.info(`[Google] Complete: ${urls.size} URLs for "${keyword}"`);
+      return Array.from(urls);
     } catch (error) {
       logger.error(`Google search error for "${keyword}"`, { message: error.message });
+      return [];
     }
+  }
 
-    logger.info(`Google search complete: found ${urls.size} URLs for "${keyword}"`);
-    return Array.from(urls);
+  /**
+   * 获取单页结果
+   */
+  async _fetchPage(keyword, start) {
+    const searchUrl = this._buildSearchUrl(keyword, start);
+    
+    const response = await withRetry(
+      () => httpClient.get(searchUrl, {
+        timeout: 10000,
+        headers: {
+          'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+          'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
+          'Accept-Language': 'en-US,en;q=0.5',
+          'Referer': 'https://www.google.com/',
+        },
+      }),
+      { maxRetries: 1, retryCondition: isRetryableError }
+    );
+    
+    if (response.status !== 200) {
+      return [];
+    }
+    
+    return this._parseResponse(response.data);
   }
 
   /**

package/src/services/orchestrator.js

@@ -18,19 +18,13 @@ import logger from '../infrastructure/logger.js';
 import config from '../config/index.js';
 import { searchCache } from '../infrastructure/cache.js';
 import { metrics } from '../infrastructure/metrics.js';
+import { requestQueue } from '../infrastructure/requestQueue.js';
 
-// 极速并发配置 - 最大化吹吐量
-const MAX_CONCURRENT_REQUESTS = 5;   // 同时处理 5 个 MCP 请求
+// 极速并发配置
 const MAX_CONCURRENT_KEYWORDS = 3;   // 每个请求内并行 3 个关键词
-const MAX_CONCURRENT_SEARCHES = 5;   // 并行搜索 5 个
 
-// 全局共享的并发限制器
-const globalRequestLimit = pLimit(MAX_CONCURRENT_REQUESTS);
+// 关键词并发限制器
 const globalKeywordLimit = pLimit(MAX_CONCURRENT_KEYWORDS);
-const globalSearchLimit = pLimit(MAX_CONCURRENT_SEARCHES);
-
-// 请求状态跟踪
-let activeRequests = new Map(); // requestId -> { startTime, query, status }
 
 export class Orchestrator {
   constructor() {
@@ -38,7 +32,6 @@ export class Orchestrator {
     this.fileManager = new FileManager();
     this.imageProcessor = new ImageProcessor();
     this.keywordLimit = globalKeywordLimit;
-    this.requestLimit = globalRequestLimit;
   }
 
   /**
@@ -256,79 +249,26 @@ export class Orchestrator {
   }
 
   /**
-   * 执行任务 - 高性能入口
-   * 支持多个 MCP 请求并行处理
+   * 执行任务 - 使用请求队列管理
+   * 支持多个 MCP 请求并行处理，自动资源释放
    * @param {Object} params - 任务参数
    * @returns {Promise<Object>} - 执行结果
    */
   async execute(params) {
-    const requestId = `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-    const pendingCount = this.requestLimit.pendingCount;
-    const activeCount = this.requestLimit.activeCount;
-    
-    // 记录请求指标
     metrics.recordRequest();
     
-    // 检查是否超过最大并发限制
-    if (activeCount >= MAX_CONCURRENT_REQUESTS && pendingCount >= MAX_CONCURRENT_REQUESTS) {
-      logger.warn(`[${requestId}] Request rejected: queue full (${activeCount} active, ${pendingCount} pending)`);
-      metrics.recordError('queue_full');
-      return {
-        success: false,
-        error: `服务繁忙，已达到最大并发数 ${MAX_CONCURRENT_REQUESTS}。请稍后重试。`,
-        requestId,
-        activeRequests: activeCount,
-        pendingRequests: pendingCount,
-      };
-    }
-    
-    // 记录活跃请求
-    activeRequests.set(requestId, {
-      startTime: Date.now(),
-      query: params.query,
-      status: 'queued',
-    });
-    
-    logger.info(`[${requestId}] Request queued: ${activeCount} active, ${pendingCount} pending, query="${params.query}"`);
-    
-    try {
-      // 使用并发限制器，支持多个请求并行
-      const result = await this.requestLimit(async () => {
-        activeRequests.get(requestId).status = 'processing';
-        logger.info(`[${requestId}] Processing started`);
-        return await this._executeInternal(params, requestId);
-      });
-      
-      result.requestId = requestId;
-      result.processingTime = Date.now() - activeRequests.get(requestId).startTime;
-      return result;
-    } catch (error) {
-      logger.error(`[${requestId}] Request failed: ${error.message}`);
-      metrics.recordError(error);
-      return {
-        success: false,
-        error: error.message,
-        requestId,
-      };
-    } finally {
-      activeRequests.delete(requestId);
-    }
+    // 使用请求队列提交任务
+    return requestQueue.submit(
+      () => this._executeInternal(params),
+      { query: params.query, mode: params.mode }
+    );
   }
   
   /**
    * 获取当前请求状态
    */
   static getStatus() {
-    return {
-      activeRequests: Array.from(activeRequests.entries()).map(([id, info]) => ({
-        id,
-        query: info.query,
-        status: info.status,
-        duration: Date.now() - info.startTime,
-      })),
-      activeCount: activeRequests.size,
-      maxConcurrent: MAX_CONCURRENT_REQUESTS,
-    };
+    return requestQueue.getStatus();
   }
 
   /**