smart-image-scraper-mcp 1.1.3 → 2.1.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smart-image-scraper-mcp",
-  "version": "1.1.3",
+  "version": "2.1.0",
   "description": "全网智能图片抓取 MCP 服务器 - 支持 Bing/Google 图片搜索、验证和下载",
   "main": "src/index.js",
   "type": "module",

package/src/infrastructure/metrics.js

@@ -120,13 +120,41 @@ export class MetricsCollector {
    * 记录错误
    */
   recordError(error) {
-    const code = error.code || 'UNKNOWN';
-    const type = error.name || 'Error';
+    const code = typeof error === 'string' ? error : (error.code || 'UNKNOWN');
+    const type = typeof error === 'string' ? error : (error.name || 'Error');
     
     this.metrics.errors.byCode[code] = (this.metrics.errors.byCode[code] || 0) + 1;
     this.metrics.errors.byType[type] = (this.metrics.errors.byType[type] || 0) + 1;
   }
 
+  /**
+   * 记录请求
+   */
+  recordRequest() {
+    this.metrics.requests.total++;
+    this.metrics.system.lastActivity = Date.now();
+  }
+
+  /**
+   * 记录缓存命中
+   */
+  recordCacheHit() {
+    if (!this.metrics.cache) {
+      this.metrics.cache = { hits: 0, misses: 0 };
+    }
+    this.metrics.cache.hits++;
+  }
+
+  /**
+   * 记录缓存未命中
+   */
+  recordCacheMiss() {
+    if (!this.metrics.cache) {
+      this.metrics.cache = { hits: 0, misses: 0 };
+    }
+    this.metrics.cache.misses++;
+  }
+
   /**
    * 添加持续时间样本
    */

package/src/providers/bingScraper.js

@@ -46,8 +46,8 @@ export class BingScraper extends BaseScraper {
           break;
         }
         
-        // 添加请求间隔，防止被封
-        await this._delay(200 + Math.random() * 100);
+        // 极速模式：最小延迟
+        await this._delay(50);
 
         const newUrls = this._parseResponse(response.data);
         

package/src/providers/googleScraper.js

@@ -74,8 +74,8 @@ export class GoogleScraper extends BaseScraper {
 
         start += pageSize;
         
-        // 添加延迟防止被封
-        await this._delay(300 + Math.random() * 200);
+        // 极速模式：最小延迟
+        await this._delay(50);
       }
     } catch (error) {
       logger.error(`Google search error for "${keyword}"`, { message: error.message });

package/src/services/linkValidator.js

@@ -8,8 +8,9 @@ import httpClient from '../infrastructure/httpClient.js';
 import logger from '../infrastructure/logger.js';
 import config from '../config/index.js';
 
-// 全局共享的并发限制器，避免多个请求同时发起导致资源竞争
-const globalValidateLimit = pLimit(5); // 限制验证并发为 5
+// 极速并发验证配置
+const MAX_VALIDATE_CONCURRENCY = 30; // 极速验证并发 30
+const globalValidateLimit = pLimit(MAX_VALIDATE_CONCURRENCY);
 
 export class LinkValidator {
   constructor() {
@@ -32,7 +33,7 @@ export class LinkValidator {
   }
 
   /**
-   * 验证单个链接
+   * 验证单个链接 - 极速版本
    * @param {string} url - 图片URL
    * @param {boolean} fetchQuality - 是否获取质量信息
    * @returns {Promise<{url: string, valid: boolean, error?: string, quality?: Object}>}
@@ -44,14 +45,12 @@ export class LinkValidator {
     }
 
     const controller = new AbortController();
-    const timeoutId = setTimeout(() => {
-      controller.abort();
-    }, 5000); // 缩短超时时间到5秒
+    const timeoutId = setTimeout(() => controller.abort(), 2000); // 极速超时 2 秒
 
     try {
       const response = await httpClient.head(url, {
-        timeout: 4000, // 缩短超时
-        maxRedirects: 2, // 减少重定向次数
+        timeout: 1500, // 极速超时 1.5 秒
+        maxRedirects: 1, // 最多 1 次重定向
         signal: controller.signal,
       });
 
@@ -59,27 +58,32 @@ export class LinkValidator {
 
       const contentType = response.headers['content-type'] || '';
       const isImage = contentType.toLowerCase().includes('image');
-      const isValidStatus = response.status === 200;
+      const isValidStatus = response.status === 200 || response.status === 206;
 
       if (isValidStatus && isImage) {
-        // 获取质量信息
         const quality = fetchQuality ? this._extractQualityInfo(response.headers, url) : null;
         return { url, valid: true, quality };
       }
 
-      // HEAD 失败不再尝试 GET，直接返回失败（加快速度）
-      return { url, valid: false, error: `Invalid: status=${response.status}` };
+      return { url, valid: false, error: `status=${response.status}` };
     } catch (error) {
       clearTimeout(timeoutId);
-      
-      // 超时或取消，直接返回失败
-      if (error.name === 'AbortError' || error.code === 'ERR_CANCELED' || error.code === 'ECONNABORTED') {
-        return { url, valid: false, error: 'Timeout' };
-      }
-      
-      // 其他错误也直接返回失败（不再尝试 GET）
-      return { url, valid: false, error: error.message || 'Request failed' };
+      return { url, valid: false, error: 'timeout' };
+    }
+  }
+  
+  /**
+   * 快速验证 - 仅检查 URL 格式，不发送 HTTP 请求
+   */
+  quickValidate(url) {
+    if (!this._isValidUrlFormat(url)) {
+      return { url, valid: false, error: 'Invalid URL' };
     }
+    // 检查常见图片扩展名
+    const imageExtensions = ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp', '.svg'];
+    const urlLower = url.toLowerCase();
+    const hasImageExt = imageExtensions.some(ext => urlLower.includes(ext));
+    return { url, valid: hasImageExt, quality: null };
   }
 
   /**

package/src/services/orchestrator.js

@@ -1,6 +1,12 @@
 /**
- * 编排器
+ * 编排器 - 高性能实现
  * 负责解析批量关键词，分配任务，协调各模块工作
+ * 
+ * 高性能特性：
+ * - 支持多个 MCP 请求并行处理
+ * - 智能负载均衡和资源分配
+ * - 缓存集成减少重复请求
+ * - 快速响应模式
  */
 
 import pLimit from 'p-limit';
@@ -10,10 +16,21 @@ import { FileManager } from './fileManager.js';
 import { ImageProcessor } from './imageProcessor.js';
 import logger from '../infrastructure/logger.js';
 import config from '../config/index.js';
+import { searchCache } from '../infrastructure/cache.js';
+import { metrics } from '../infrastructure/metrics.js';
 
-// 全局共享的并发限制器，避免多个 MCP 请求同时发起导致资源竞争
-const globalKeywordLimit = pLimit(1); // 严格限制关键词并发
-const globalRequestLimit = pLimit(1); // 严格限制同时只处理 1 个 MCP 请求
+// 极速并发配置 - 最大化吹吐量
+const MAX_CONCURRENT_REQUESTS = 5;   // 同时处理 5 个 MCP 请求
+const MAX_CONCURRENT_KEYWORDS = 3;   // 每个请求内并行 3 个关键词
+const MAX_CONCURRENT_SEARCHES = 5;   // 并行搜索 5 个
+
+// 全局共享的并发限制器
+const globalRequestLimit = pLimit(MAX_CONCURRENT_REQUESTS);
+const globalKeywordLimit = pLimit(MAX_CONCURRENT_KEYWORDS);
+const globalSearchLimit = pLimit(MAX_CONCURRENT_SEARCHES);
+
+// 请求状态跟踪
+let activeRequests = new Map(); // requestId -> { startTime, query, status }
 
 export class Orchestrator {
   constructor() {
@@ -50,20 +67,29 @@ export class Orchestrator {
    */
   async processKeywordLink(keyword, count, source, options = {}) {
     const startTime = Date.now();
-    // Link 模式默认不进行质量评估（加快速度），除非明确要求
+    const fastMode = options.fastMode !== false; // 默认开启快速模式
     const prioritizeQuality = options.prioritizeQuality === true;
     
     try {
-      // 获取搜索源
       const scraper = getScraper(source);
+      const searchCount = fastMode ? count : Math.ceil(count * 1.5);
       
-      logger.info(`Searching for "${keyword}"...`);
-      
-      // 搜索图片（多获取一些以弥补验证失败的损失）
-      const searchCount = Math.ceil(count * 1.5);
-      const rawUrls = await scraper.search(keyword, searchCount, options);
+      // 尝试从缓存获取
+      const cachedUrls = searchCache.getSearchResult(keyword, source, options);
+      let rawUrls;
       
-      logger.info(`Found ${rawUrls.length} URLs for "${keyword}"`);
+      if (cachedUrls && cachedUrls.length >= searchCount) {
+        logger.info(`[CACHE] "${keyword}" - ${cachedUrls.length} URLs`);
+        rawUrls = cachedUrls;
+        metrics.recordCacheHit();
+      } else {
+        logger.info(`[SEARCH] "${keyword}"...`);
+        rawUrls = await scraper.search(keyword, searchCount, options);
+        if (rawUrls.length > 0) {
+          searchCache.setSearchResult(keyword, source, options, rawUrls);
+        }
+        metrics.recordCacheMiss();
+      }
       
       if (rawUrls.length === 0) {
         return {
@@ -74,25 +100,30 @@ export class Orchestrator {
         };
       }
 
-      // 验证链接
-      const { valid, invalid } = await this.linkValidator.validateMany(rawUrls, {
-        fetchQuality: prioritizeQuality,
-        sortByQuality: prioritizeQuality,
-      });
-      
-      // 截取需要的数量
-      const resultUrls = valid.slice(0, count).map(v => v.url);
+      // 快速模式：跳过 HTTP 验证，仅检查 URL 格式
+      let resultUrls;
+      if (fastMode && !prioritizeQuality) {
+        // 快速验证：仅检查 URL 格式和扩展名
+        const quickResults = rawUrls.map(url => this.linkValidator.quickValidate(url));
+        resultUrls = quickResults.filter(r => r.valid).slice(0, count).map(r => r.url);
+        logger.info(`[FAST] "${keyword}" - ${resultUrls.length} URLs (no HTTP validation)`);
+      } else {
+        // 完整验证
+        const { valid } = await this.linkValidator.validateMany(rawUrls, {
+          fetchQuality: prioritizeQuality,
+          sortByQuality: prioritizeQuality,
+        });
+        resultUrls = valid.slice(0, count).map(v => v.url);
+      }
 
       return {
         keyword,
         success: true,
         mode: 'link',
         totalSearched: rawUrls.length,
-        totalValidated: valid.length,
-        totalInvalid: invalid.length,
         urls: resultUrls,
         count: resultUrls.length,
-        qualitySorted: prioritizeQuality,
+        fastMode,
         duration: Date.now() - startTime,
       };
     } catch (error) {
@@ -116,15 +147,28 @@ export class Orchestrator {
    */
   async processKeywordDownload(keyword, count, source, options = {}) {
     const startTime = Date.now();
-    const prioritizeQuality = options.prioritizeQuality !== false; // 默认优先高质量
+    const prioritizeQuality = options.prioritizeQuality !== false;
     
     try {
-      // 获取搜索源
       const scraper = getScraper(source);
-      
-      // 搜索图片（多获取一些以弥补下载失败的损失）
       const searchCount = Math.ceil(count * 2);
-      const rawUrls = await scraper.search(keyword, searchCount, options);
+      
+      // 尝试从缓存获取
+      const cachedUrls = searchCache.getSearchResult(keyword, source, options);
+      let rawUrls;
+      
+      if (cachedUrls && cachedUrls.length >= searchCount) {
+        logger.info(`[CACHE HIT] "${keyword}" - ${cachedUrls.length} URLs from cache`);
+        rawUrls = cachedUrls;
+        metrics.recordCacheHit();
+      } else {
+        logger.info(`[SEARCH] "${keyword}"...`);
+        rawUrls = await scraper.search(keyword, searchCount, options);
+        if (rawUrls.length > 0) {
+          searchCache.setSearchResult(keyword, source, options, rawUrls);
+        }
+        metrics.recordCacheMiss();
+      }
       
       if (rawUrls.length === 0) {
         return {
@@ -207,26 +251,79 @@ export class Orchestrator {
   }
 
   /**
-   * 执行任务
+   * 执行任务 - 高性能入口
+   * 支持多个 MCP 请求并行处理
    * @param {Object} params - 任务参数
    * @returns {Promise<Object>} - 执行结果
    */
   async execute(params) {
-    // 检查当前队列状态
+    const requestId = `req_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
     const pendingCount = this.requestLimit.pendingCount;
     const activeCount = this.requestLimit.activeCount;
     
-    // 如果队列中已有等待的请求，直接返回提示（避免长时间等待）
-    if (pendingCount > 0) {
-      logger.warn(`Request queued: ${pendingCount} pending, ${activeCount} active`);
+    // 记录请求指标
+    metrics.recordRequest();
+    
+    // 检查是否超过最大并发限制
+    if (activeCount >= MAX_CONCURRENT_REQUESTS && pendingCount >= MAX_CONCURRENT_REQUESTS) {
+      logger.warn(`[${requestId}] Request rejected: queue full (${activeCount} active, ${pendingCount} pending)`);
+      metrics.recordError('queue_full');
       return {
         success: false,
-        error: `当前有 ${pendingCount + activeCount} 个请求正在处理中，请稍后重试。建议一次只发起一个搜索请求。`,
+        error: `服务繁忙，已达到最大并发数 ${MAX_CONCURRENT_REQUESTS}。请稍后重试。`,
+        requestId,
+        activeRequests: activeCount,
+        pendingRequests: pendingCount,
       };
     }
     
-    // 使用全局请求限制器
-    return this.requestLimit(() => this._executeInternal(params));
+    // 记录活跃请求
+    activeRequests.set(requestId, {
+      startTime: Date.now(),
+      query: params.query,
+      status: 'queued',
+    });
+    
+    logger.info(`[${requestId}] Request queued: ${activeCount} active, ${pendingCount} pending, query="${params.query}"`);
+    
+    try {
+      // 使用并发限制器，支持多个请求并行
+      const result = await this.requestLimit(async () => {
+        activeRequests.get(requestId).status = 'processing';
+        logger.info(`[${requestId}] Processing started`);
+        return await this._executeInternal(params, requestId);
+      });
+      
+      result.requestId = requestId;
+      result.processingTime = Date.now() - activeRequests.get(requestId).startTime;
+      return result;
+    } catch (error) {
+      logger.error(`[${requestId}] Request failed: ${error.message}`);
+      metrics.recordError(error);
+      return {
+        success: false,
+        error: error.message,
+        requestId,
+      };
+    } finally {
+      activeRequests.delete(requestId);
+    }
+  }
+  
+  /**
+   * 获取当前请求状态
+   */
+  static getStatus() {
+    return {
+      activeRequests: Array.from(activeRequests.entries()).map(([id, info]) => ({
+        id,
+        query: info.query,
+        status: info.status,
+        duration: Date.now() - info.startTime,
+      })),
+      activeCount: activeRequests.size,
+      maxConcurrent: MAX_CONCURRENT_REQUESTS,
+    };
   }
 
   /**