smart-image-scraper-mcp 1.1.2 → 2.0.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smart-image-scraper-mcp",
-  "version": "1.1.2",
+  "version": "2.0.0",
   "description": "全网智能图片抓取 MCP 服务器 - 支持 Bing/Google 图片搜索、验证和下载",
   "main": "src/index.js",
   "type": "module",

package/src/infrastructure/metrics.js

@@ -120,13 +120,41 @@ export class MetricsCollector {
    * 记录错误
    */
   recordError(error) {
-    const code = error.code || 'UNKNOWN';
-    const type = error.name || 'Error';
+    const code = typeof error === 'string' ? error : (error.code || 'UNKNOWN');
+    const type = typeof error === 'string' ? error : (error.name || 'Error');
     
     this.metrics.errors.byCode[code] = (this.metrics.errors.byCode[code] || 0) + 1;
     this.metrics.errors.byType[type] = (this.metrics.errors.byType[type] || 0) + 1;
   }
 
+  /**
+   * 记录请求
+   */
+  recordRequest() {
+    this.metrics.requests.total++;
+    this.metrics.system.lastActivity = Date.now();
+  }
+
+  /**
+   * 记录缓存命中
+   */
+  recordCacheHit() {
+    if (!this.metrics.cache) {
+      this.metrics.cache = { hits: 0, misses: 0 };
+    }
+    this.metrics.cache.hits++;
+  }
+
+  /**
+   * 记录缓存未命中
+   */
+  recordCacheMiss() {
+    if (!this.metrics.cache) {
+      this.metrics.cache = { hits: 0, misses: 0 };
+    }
+    this.metrics.cache.misses++;
+  }
+
   /**
    * 添加持续时间样本
    */

package/src/providers/bingScraper.js

@@ -46,8 +46,8 @@ export class BingScraper extends BaseScraper {
           break;
         }
         
-        // 添加请求间隔，防止被封
-        await this._delay(200 + Math.random() * 100);
+        // 减少请求间隔，提高速度
+        await this._delay(100 + Math.random() * 50);
 
         const newUrls = this._parseResponse(response.data);
         

package/src/providers/googleScraper.js

@@ -74,8 +74,8 @@ export class GoogleScraper extends BaseScraper {
 
         start += pageSize;
         
-        // 添加延迟防止被封
-        await this._delay(300 + Math.random() * 200);
+        // 减少延迟，提高速度
+        await this._delay(100 + Math.random() * 100);
       }
     } catch (error) {
       logger.error(`Google search error for "${keyword}"`, { message: error.message });

package/src/services/linkValidator.js

@@ -8,8 +8,9 @@ import httpClient from '../infrastructure/httpClient.js';
 import logger from '../infrastructure/logger.js';
 import config from '../config/index.js';
 
-// 全局共享的并发限制器，避免多个请求同时发起导致资源竞争
-const globalValidateLimit = pLimit(5); // 限制验证并发为 5
+// 高性能并发验证配置
+const MAX_VALIDATE_CONCURRENCY = 15; // 提高验证并发到 15
+const globalValidateLimit = pLimit(MAX_VALIDATE_CONCURRENCY);
 
 export class LinkValidator {
   constructor() {

package/src/services/orchestrator.js

@@ -1,6 +1,12 @@
 /**
- * 编排器
+ * 编排器 - 高性能实现
  * 负责解析批量关键词，分配任务，协调各模块工作
+ * 
+ * 高性能特性：
+ * - 支持多个 MCP 请求并行处理
+ * - 智能负载均衡和资源分配
+ * - 缓存集成减少重复请求
+ * - 快速响应模式
  */
 
 import pLimit from 'p-limit';
@@ -10,10 +16,21 @@ import { FileManager } from './fileManager.js';
 import { ImageProcessor } from './imageProcessor.js';
 import logger from '../infrastructure/logger.js';
 import config from '../config/index.js';
+import { searchCache } from '../infrastructure/cache.js';
+import { metrics } from '../infrastructure/metrics.js';
 
-// 全局共享的并发限制器，避免多个 MCP 请求同时发起导致资源竞争
-const globalKeywordLimit = pLimit(1); // 严格限制关键词并发
-const globalRequestLimit = pLimit(1); // 严格限制同时只处理 1 个 MCP 请求
+// 高性能并发配置
+const MAX_CONCURRENT_REQUESTS = 3;  // 同时处理最多 3 个 MCP 请求
+const MAX_CONCURRENT_KEYWORDS = 2;  // 每个请求内并行处理 2 个关键词
+const MAX_CONCURRENT_VALIDATIONS = 10; // 并行验证 10 个 URL
+
+// 全局共享的并发限制器
+const globalRequestLimit = pLimit(MAX_CONCURRENT_REQUESTS);
+const globalKeywordLimit = pLimit(MAX_CONCURRENT_KEYWORDS);
+const globalValidationLimit = pLimit(MAX_CONCURRENT_VALIDATIONS);
+
+// 请求状态跟踪
+let activeRequests = new Map(); // requestId -> { startTime, query, status }
 
 export class Orchestrator {
   constructor() {
@@ -50,18 +67,30 @@ export class Orchestrator {
    */
   async processKeywordLink(keyword, count, source, options = {}) {
     const startTime = Date.now();
-    // Link 模式默认不进行质量评估（加快速度），除非明确要求
     const prioritizeQuality = options.prioritizeQuality === true;
     
     try {
-      // 获取搜索源
       const scraper = getScraper(source);
+      const searchCount = Math.ceil(count * 1.5);
       
-      logger.info(`Searching for "${keyword}"...`);
+      // 尝试从缓存获取
+      const cachedUrls = searchCache.getSearchResult(keyword, source, options);
+      let rawUrls;
       
-      // 搜索图片（多获取一些以弥补验证失败的损失）
-      const searchCount = Math.ceil(count * 1.5);
-      const rawUrls = await scraper.search(keyword, searchCount, options);
+      if (cachedUrls && cachedUrls.length >= searchCount) {
+        logger.info(`[CACHE HIT] "${keyword}" - ${cachedUrls.length} URLs from cache`);
+        rawUrls = cachedUrls;
+        metrics.recordCacheHit();
+      } else {
+        logger.info(`[SEARCH] "${keyword}"...`);
+        rawUrls = await scraper.search(keyword, searchCount, options);
+        
+        // 缓存搜索结果
+        if (rawUrls.length > 0) {
+          searchCache.setSearchResult(keyword, source, options, rawUrls);
+        }
+        metrics.recordCacheMiss();
+      }
       
       logger.info(`Found ${rawUrls.length} URLs for "${keyword}"`);
       
@@ -116,15 +145,28 @@ export class Orchestrator {
    */
   async processKeywordDownload(keyword, count, source, options = {}) {
     const startTime = Date.now();
-    const prioritizeQuality = options.prioritizeQuality !== false; // 默认优先高质量
+    const prioritizeQuality = options.prioritizeQuality !== false;
     
     try {
-      // 获取搜索源
       const scraper = getScraper(source);
-      
-      // 搜索图片（多获取一些以弥补下载失败的损失）
       const searchCount = Math.ceil(count * 2);
-      const rawUrls = await scraper.search(keyword, searchCount, options);
+      
+      // 尝试从缓存获取
+      const cachedUrls = searchCache.getSearchResult(keyword, source, options);
+      let rawUrls;
+      
+      if (cachedUrls && cachedUrls.length >= searchCount) {
+        logger.info(`[CACHE HIT] "${keyword}" - ${cachedUrls.length} URLs from cache`);
+        rawUrls = cachedUrls;
+        metrics.recordCacheHit();
+      } else {
+        logger.info(`[SEARCH] "${keyword}"...`);
+        rawUrls = await scraper.search(keyword, searchCount, options);
+        if (rawUrls.length > 0) {
+          searchCache.setSearchResult(keyword, source, options, rawUrls);
+        }
+        metrics.recordCacheMiss();
+      }
       
       if (rawUrls.length === 0) {
         return {
@@ -207,13 +249,79 @@ export class Orchestrator {
   }
 
   /**
-   * 执行任务
+   * 执行任务 - 高性能入口
+   * 支持多个 MCP 请求并行处理
    * @param {Object} params - 任务参数
    * @returns {Promise<Object>} - 执行结果
    */
   async execute(params) {
-    // 使用全局请求限制器，避免多个 MCP 请求同时执行
-    return this.requestLimit(() => this._executeInternal(params));
+    const requestId = `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
+    const pendingCount = this.requestLimit.pendingCount;
+    const activeCount = this.requestLimit.activeCount;
+    
+    // 记录请求指标
+    metrics.recordRequest();
+    
+    // 检查是否超过最大并发限制
+    if (activeCount >= MAX_CONCURRENT_REQUESTS && pendingCount >= MAX_CONCURRENT_REQUESTS) {
+      logger.warn(`[${requestId}] Request rejected: queue full (${activeCount} active, ${pendingCount} pending)`);
+      metrics.recordError('queue_full');
+      return {
+        success: false,
+        error: `服务繁忙，已达到最大并发数 ${MAX_CONCURRENT_REQUESTS}。请稍后重试。`,
+        requestId,
+        activeRequests: activeCount,
+        pendingRequests: pendingCount,
+      };
+    }
+    
+    // 记录活跃请求
+    activeRequests.set(requestId, {
+      startTime: Date.now(),
+      query: params.query,
+      status: 'queued',
+    });
+    
+    logger.info(`[${requestId}] Request queued: ${activeCount} active, ${pendingCount} pending, query="${params.query}"`);
+    
+    try {
+      // 使用并发限制器，支持多个请求并行
+      const result = await this.requestLimit(async () => {
+        activeRequests.get(requestId).status = 'processing';
+        logger.info(`[${requestId}] Processing started`);
+        return await this._executeInternal(params, requestId);
+      });
+      
+      result.requestId = requestId;
+      result.processingTime = Date.now() - activeRequests.get(requestId).startTime;
+      return result;
+    } catch (error) {
+      logger.error(`[${requestId}] Request failed: ${error.message}`);
+      metrics.recordError(error);
+      return {
+        success: false,
+        error: error.message,
+        requestId,
+      };
+    } finally {
+      activeRequests.delete(requestId);
+    }
+  }
+  
+  /**
+   * 获取当前请求状态
+   */
+  static getStatus() {
+    return {
+      activeRequests: Array.from(activeRequests.entries()).map(([id, info]) => ({
+        id,
+        query: info.query,
+        status: info.status,
+        duration: Date.now() - info.startTime,
+      })),
+      activeCount: activeRequests.size,
+      maxConcurrent: MAX_CONCURRENT_REQUESTS,
+    };
   }
 
   /**