smart-image-scraper-mcp 2.1.0 → 2.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smart-image-scraper-mcp",
-  "version": "2.1.0",
+  "version": "2.2.0",
   "description": "全网智能图片抓取 MCP 服务器 - 支持 Bing/Google 图片搜索、验证和下载",
   "main": "src/index.js",
   "type": "module",

package/src/infrastructure/requestQueue.js

@@ -0,0 +1,260 @@
+/**
+ * 请求队列管理器 - 多线程队列机制
+ * 管理并发请求，确保资源正确释放
+ */
+
+import logger from './logger.js';
+import { metrics } from './metrics.js';
+
+/**
+ * 请求队列管理器
+ */
+export class RequestQueue {
+  constructor(options = {}) {
+    this.maxConcurrent = options.maxConcurrent || 5;  // 最大并发数
+    this.maxQueueSize = options.maxQueueSize || 20;   // 最大队列长度
+    this.requestTimeout = options.requestTimeout || 60000; // 请求超时 60 秒
+    
+    this.queue = [];           // 等待队列
+    this.active = new Map();   // 活跃请求 Map<requestId, requestInfo>
+    this.completed = [];       // 已完成请求（保留最近 100 个）
+    this.maxCompleted = 100;
+    
+    this.stats = {
+      totalProcessed: 0,
+      totalSuccess: 0,
+      totalFailed: 0,
+      totalTimeout: 0,
+    };
+    
+    // 定期清理超时请求
+    this.cleanupInterval = setInterval(() => this._cleanupTimeouts(), 5000);
+  }
+
+  /**
+   * 生成请求 ID
+   */
+  generateRequestId() {
+    return `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+  }
+
+  /**
+   * 提交请求到队列
+   * @param {Function} task - 异步任务函数
+   * @param {Object} metadata - 请求元数据
+   * @returns {Promise<Object>} - 任务结果
+   */
+  async submit(task, metadata = {}) {
+    const requestId = this.generateRequestId();
+    
+    // 检查队列是否已满
+    if (this.queue.length >= this.maxQueueSize) {
+      logger.warn(`[Queue] Request rejected: queue full (${this.queue.length}/${this.maxQueueSize})`);
+      metrics.recordError('queue_full');
+      return {
+        success: false,
+        error: `队列已满，请稍后重试。当前队列: ${this.queue.length}`,
+        requestId,
+        queueStatus: this.getStatus(),
+      };
+    }
+
+    // 创建请求包装
+    const requestInfo = {
+      id: requestId,
+      metadata,
+      createdAt: Date.now(),
+      status: 'queued',
+      resolve: null,
+      reject: null,
+    };
+
+    // 创建 Promise
+    const promise = new Promise((resolve, reject) => {
+      requestInfo.resolve = resolve;
+      requestInfo.reject = reject;
+      requestInfo.task = task;
+    });
+
+    // 加入队列
+    this.queue.push(requestInfo);
+    logger.info(`[Queue] Request queued: ${requestId} (queue: ${this.queue.length}, active: ${this.active.size})`);
+    
+    // 尝试处理队列
+    this._processQueue();
+
+    // 设置超时
+    const timeoutPromise = new Promise((_, reject) => {
+      setTimeout(() => {
+        reject(new Error('Request timeout'));
+      }, this.requestTimeout);
+    });
+
+    try {
+      const result = await Promise.race([promise, timeoutPromise]);
+      return result;
+    } catch (error) {
+      if (error.message === 'Request timeout') {
+        this.stats.totalTimeout++;
+        this._removeFromActive(requestId);
+        return {
+          success: false,
+          error: '请求超时',
+          requestId,
+        };
+      }
+      throw error;
+    }
+  }
+
+  /**
+   * 处理队列中的请求
+   */
+  _processQueue() {
+    while (this.queue.length > 0 && this.active.size < this.maxConcurrent) {
+      const requestInfo = this.queue.shift();
+      this._executeRequest(requestInfo);
+    }
+  }
+
+  /**
+   * 执行单个请求
+   */
+  async _executeRequest(requestInfo) {
+    const { id, task, resolve, metadata } = requestInfo;
+    
+    // 移入活跃列表
+    requestInfo.status = 'processing';
+    requestInfo.startedAt = Date.now();
+    this.active.set(id, requestInfo);
+    
+    logger.info(`[Queue] Processing: ${id} (active: ${this.active.size})`);
+
+    try {
+      // 执行任务
+      const result = await task();
+      
+      // 标记完成
+      requestInfo.status = 'completed';
+      requestInfo.completedAt = Date.now();
+      requestInfo.duration = requestInfo.completedAt - requestInfo.startedAt;
+      
+      this.stats.totalProcessed++;
+      this.stats.totalSuccess++;
+      
+      // 添加请求信息到结果
+      result.requestId = id;
+      result.processingTime = requestInfo.duration;
+      
+      resolve(result);
+      
+      logger.info(`[Queue] Completed: ${id} in ${requestInfo.duration}ms`);
+    } catch (error) {
+      requestInfo.status = 'failed';
+      requestInfo.error = error.message;
+      
+      this.stats.totalProcessed++;
+      this.stats.totalFailed++;
+      
+      resolve({
+        success: false,
+        error: error.message,
+        requestId: id,
+      });
+      
+      logger.error(`[Queue] Failed: ${id} - ${error.message}`);
+    } finally {
+      // 从活跃列表移除
+      this._removeFromActive(id);
+      
+      // 保存到已完成列表
+      this._addToCompleted(requestInfo);
+      
+      // 继续处理队列
+      this._processQueue();
+    }
+  }
+
+  /**
+   * 从活跃列表移除
+   */
+  _removeFromActive(requestId) {
+    this.active.delete(requestId);
+  }
+
+  /**
+   * 添加到已完成列表
+   */
+  _addToCompleted(requestInfo) {
+    this.completed.push({
+      id: requestInfo.id,
+      status: requestInfo.status,
+      duration: requestInfo.duration,
+      completedAt: requestInfo.completedAt,
+    });
+    
+    // 限制已完成列表大小
+    if (this.completed.length > this.maxCompleted) {
+      this.completed.shift();
+    }
+  }
+
+  /**
+   * 清理超时请求
+   */
+  _cleanupTimeouts() {
+    const now = Date.now();
+    
+    for (const [id, info] of this.active.entries()) {
+      if (now - info.startedAt > this.requestTimeout) {
+        logger.warn(`[Queue] Timeout cleanup: ${id}`);
+        info.reject(new Error('Request timeout'));
+        this._removeFromActive(id);
+        this.stats.totalTimeout++;
+      }
+    }
+  }
+
+  /**
+   * 获取队列状态
+   */
+  getStatus() {
+    return {
+      queueLength: this.queue.length,
+      activeCount: this.active.size,
+      maxConcurrent: this.maxConcurrent,
+      maxQueueSize: this.maxQueueSize,
+      stats: { ...this.stats },
+      activeRequests: Array.from(this.active.entries()).map(([id, info]) => ({
+        id,
+        status: info.status,
+        duration: Date.now() - info.startedAt,
+        metadata: info.metadata,
+      })),
+    };
+  }
+
+  /**
+   * 清理资源
+   */
+  cleanup() {
+    clearInterval(this.cleanupInterval);
+    
+    // 拒绝所有等待中的请求
+    for (const info of this.queue) {
+      info.reject(new Error('Queue shutdown'));
+    }
+    this.queue = [];
+    
+    logger.info('[Queue] Cleanup completed');
+  }
+}
+
+// 全局请求队列实例
+export const requestQueue = new RequestQueue({
+  maxConcurrent: 5,
+  maxQueueSize: 20,
+  requestTimeout: 60000,
+});
+
+export default requestQueue;

package/src/services/linkValidator.js

@@ -74,16 +74,15 @@ export class LinkValidator {
   
   /**
    * 快速验证 - 仅检查 URL 格式，不发送 HTTP 请求
+   * 宽松模式：只要 URL 格式正确就通过
    */
   quickValidate(url) {
     if (!this._isValidUrlFormat(url)) {
       return { url, valid: false, error: 'Invalid URL' };
     }
-    // 检查常见图片扩展名
-    const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.svg'];
-    const urlLower = url.toLowerCase();
-    const hasImageExt = imageExtensions.some(ext => urlLower.includes(ext));
-    return { url, valid: hasImageExt, quality: null };
+    // 宽松验证：只要 URL 格式正确就认为有效
+    // 图片 URL 可能不包含扩展名（动态生成的 URL）
+    return { url, valid: true, quality: null };
   }
 
   /**

package/src/services/orchestrator.js

@@ -18,19 +18,13 @@ import logger from '../infrastructure/logger.js';
 import config from '../config/index.js';
 import { searchCache } from '../infrastructure/cache.js';
 import { metrics } from '../infrastructure/metrics.js';
+import { requestQueue } from '../infrastructure/requestQueue.js';
 
-// 极速并发配置 - 最大化吹吐量
-const MAX_CONCURRENT_REQUESTS = 5;   // 同时处理 5 个 MCP 请求
+// 极速并发配置
 const MAX_CONCURRENT_KEYWORDS = 3;   // 每个请求内并行 3 个关键词
-const MAX_CONCURRENT_SEARCHES = 5;   // 并行搜索 5 个
 
-// 全局共享的并发限制器
-const globalRequestLimit = pLimit(MAX_CONCURRENT_REQUESTS);
+// 关键词并发限制器
 const globalKeywordLimit = pLimit(MAX_CONCURRENT_KEYWORDS);
-const globalSearchLimit = pLimit(MAX_CONCURRENT_SEARCHES);
-
-// 请求状态跟踪
-let activeRequests = new Map(); // requestId -> { startTime, query, status }
 
 export class Orchestrator {
   constructor() {
@@ -38,7 +32,6 @@ export class Orchestrator {
     this.fileManager = new FileManager();
     this.imageProcessor = new ImageProcessor();
     this.keywordLimit = globalKeywordLimit;
-    this.requestLimit = globalRequestLimit;
   }
 
   /**
@@ -67,23 +60,24 @@ export class Orchestrator {
    */
   async processKeywordLink(keyword, count, source, options = {}) {
     const startTime = Date.now();
-    const fastMode = options.fastMode !== false; // 默认开启快速模式
+    const fastMode = options.fastMode !== false;
     const prioritizeQuality = options.prioritizeQuality === true;
     
     try {
       const scraper = getScraper(source);
-      const searchCount = fastMode ? count : Math.ceil(count * 1.5);
+      // 多搜索一些以确保有足够的结果
+      const searchCount = Math.max(count * 3, 10);
       
       // 尝试从缓存获取
       const cachedUrls = searchCache.getSearchResult(keyword, source, options);
       let rawUrls;
       
-      if (cachedUrls && cachedUrls.length >= searchCount) {
+      if (cachedUrls && cachedUrls.length >= count) {
         logger.info(`[CACHE] "${keyword}" - ${cachedUrls.length} URLs`);
         rawUrls = cachedUrls;
         metrics.recordCacheHit();
       } else {
-        logger.info(`[SEARCH] "${keyword}"...`);
+        logger.info(`[SEARCH] "${keyword}" (target: ${searchCount})...`);
         rawUrls = await scraper.search(keyword, searchCount, options);
         if (rawUrls.length > 0) {
           searchCache.setSearchResult(keyword, source, options, rawUrls);
@@ -100,20 +94,24 @@ export class Orchestrator {
         };
       }
 
-      // 快速模式：跳过 HTTP 验证，仅检查 URL 格式
+      // 快速模式：直接返回搜索结果（不验证）
       let resultUrls;
       if (fastMode && !prioritizeQuality) {
-        // 快速验证：仅检查 URL 格式和扩展名
-        const quickResults = rawUrls.map(url => this.linkValidator.quickValidate(url));
-        resultUrls = quickResults.filter(r => r.valid).slice(0, count).map(r => r.url);
-        logger.info(`[FAST] "${keyword}" - ${resultUrls.length} URLs (no HTTP validation)`);
+        // 快速模式：直接使用搜索结果
+        resultUrls = rawUrls.slice(0, count);
+        logger.info(`[FAST] "${keyword}" - ${resultUrls.length} URLs`);
       } else {
-        // 完整验证
+        // 完整验证模式：验证不通过的继续搜索更多
         const { valid } = await this.linkValidator.validateMany(rawUrls, {
           fetchQuality: prioritizeQuality,
           sortByQuality: prioritizeQuality,
         });
         resultUrls = valid.slice(0, count).map(v => v.url);
+        
+        // 如果验证通过的不够，记录警告
+        if (resultUrls.length < count) {
+          logger.warn(`[VALIDATE] "${keyword}" - only ${resultUrls.length}/${count} valid`);
+        }
       }
 
       return {
@@ -251,79 +249,26 @@ export class Orchestrator {
   }
 
   /**
-   * 执行任务 - 高性能入口
-   * 支持多个 MCP 请求并行处理
+   * 执行任务 - 使用请求队列管理
+   * 支持多个 MCP 请求并行处理，自动资源释放
    * @param {Object} params - 任务参数
    * @returns {Promise<Object>} - 执行结果
    */
   async execute(params) {
-    const requestId = `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
-    const pendingCount = this.requestLimit.pendingCount;
-    const activeCount = this.requestLimit.activeCount;
-    
-    // 记录请求指标
     metrics.recordRequest();
     
-    // 检查是否超过最大并发限制
-    if (activeCount >= MAX_CONCURRENT_REQUESTS && pendingCount >= MAX_CONCURRENT_REQUESTS) {
-      logger.warn(`[${requestId}] Request rejected: queue full (${activeCount} active, ${pendingCount} pending)`);
-      metrics.recordError('queue_full');
-      return {
-        success: false,
-        error: `服务繁忙，已达到最大并发数 ${MAX_CONCURRENT_REQUESTS}。请稍后重试。`,
-        requestId,
-        activeRequests: activeCount,
-        pendingRequests: pendingCount,
-      };
-    }
-    
-    // 记录活跃请求
-    activeRequests.set(requestId, {
-      startTime: Date.now(),
-      query: params.query,
-      status: 'queued',
-    });
-    
-    logger.info(`[${requestId}] Request queued: ${activeCount} active, ${pendingCount} pending, query="${params.query}"`);
-    
-    try {
-      // 使用并发限制器，支持多个请求并行
-      const result = await this.requestLimit(async () => {
-        activeRequests.get(requestId).status = 'processing';
-        logger.info(`[${requestId}] Processing started`);
-        return await this._executeInternal(params, requestId);
-      });
-      
-      result.requestId = requestId;
-      result.processingTime = Date.now() - activeRequests.get(requestId).startTime;
-      return result;
-    } catch (error) {
-      logger.error(`[${requestId}] Request failed: ${error.message}`);
-      metrics.recordError(error);
-      return {
-        success: false,
-        error: error.message,
-        requestId,
-      };
-    } finally {
-      activeRequests.delete(requestId);
-    }
+    // 使用请求队列提交任务
+    return requestQueue.submit(
+      () => this._executeInternal(params),
+      { query: params.query, mode: params.mode }
+    );
   }
   
   /**
    * 获取当前请求状态
    */
   static getStatus() {
-    return {
-      activeRequests: Array.from(activeRequests.entries()).map(([id, info]) => ({
-        id,
-        query: info.query,
-        status: info.status,
-        duration: Date.now() - info.startTime,
-      })),
-      activeCount: activeRequests.size,
-      maxConcurrent: MAX_CONCURRENT_REQUESTS,
-    };
+    return requestQueue.getStatus();
   }
 
   /**