smart-image-scraper-mcp 2.5.2 → 2.6.0

package/src/services/linkValidator.js

@@ -8,8 +8,8 @@ import httpClient from '../infrastructure/httpClient.js';
 import logger from '../infrastructure/logger.js';
 import config from '../config/index.js';
 
-// 极速并发验证配置
-const MAX_VALIDATE_CONCURRENCY = 30; // 极速验证并发 30
+// 并发验证配置 - 降低并发避免连接池耗尽
+const MAX_VALIDATE_CONCURRENCY = 15; // 验证并发 15
 const globalValidateLimit = pLimit(MAX_VALIDATE_CONCURRENCY);
 
 export class LinkValidator {
@@ -45,11 +45,11 @@ export class LinkValidator {
     }
 
     const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 2000); // 极速超时 2 秒
+    const timeoutId = setTimeout(() => controller.abort(), 3000); // 超时 3 秒
 
     try {
       const response = await httpClient.head(url, {
-        timeout: 1500, // 极速超时 1.5 秒
+        timeout: 2500, // 超时 2.5 秒
         maxRedirects: 1, // 最多 1 次重定向
         signal: controller.signal,
       });
@@ -68,6 +68,10 @@ export class LinkValidator {
       return { url, valid: false, error: `status=${response.status}` };
     } catch (error) {
       clearTimeout(timeoutId);
+      // 确保 abort controller 被清理
+      if (!controller.signal.aborted) {
+        controller.abort();
+      }
       return { url, valid: false, error: 'timeout' };
     }
   }
@@ -178,12 +182,12 @@ export class LinkValidator {
    */
   async _validateWithGet(url, fetchQuality = false) {
     const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 8000);
+    const timeoutId = setTimeout(() => controller.abort(), 5000);
     
     try {
       const response = await httpClient.get(url, {
-        timeout: 5000,
-        maxRedirects: 3,
+        timeout: 4000,
+        maxRedirects: 2,
         responseType: 'arraybuffer',
         signal: controller.signal,
         headers: {
@@ -205,7 +209,10 @@ export class LinkValidator {
       return { url, valid: false, error: `GET validation failed: status=${response.status}` };
     } catch (error) {
       clearTimeout(timeoutId);
-      
+      // 确保 abort controller 被清理
+      if (!controller.signal.aborted) {
+        controller.abort();
+      }
       if (error.name === 'AbortError' || error.code === 'ERR_CANCELED') {
         return { url, valid: false, error: 'Request timeout' };
       }

package/src/services/orchestrator.js

@@ -20,8 +20,8 @@ import { searchCache } from '../infrastructure/cache.js';
 import { metrics } from '../infrastructure/metrics.js';
 import { requestQueue } from '../infrastructure/requestQueue.js';
 
-// 极速并发配置
-const MAX_CONCURRENT_KEYWORDS = 3;   // 每个请求内并行 3 个关键词
+// 并发配置 - 降低并发避免资源耗尽
+const MAX_CONCURRENT_KEYWORDS = 2;   // 每个请求内并行 2 个关键词
 
 // 关键词并发限制器
 const globalKeywordLimit = pLimit(MAX_CONCURRENT_KEYWORDS);
@@ -68,9 +68,8 @@ export class Orchestrator {
       // 多搜索一些以确保有足够的结果
       const searchCount = Math.max(count * 3, 10);
       
-      // 使用统一的缓存键策略
-      const cacheKey = { keyword, source, size: options.size, aspect: options.aspect };
-      const cachedUrls = searchCache.getSearchResult(keyword, source, cacheKey);
+      // 使用统一的缓存键策略（options 已包含 size, aspect, safeSearch）
+      const cachedUrls = searchCache.getSearchResult(keyword, source, options);
       let rawUrls;
       
       if (cachedUrls && cachedUrls.length >= count) {
@@ -81,7 +80,7 @@ export class Orchestrator {
         logger.info(`[SEARCH] "${keyword}" (target: ${searchCount})...`);
         rawUrls = await scraper.search(keyword, searchCount, options);
         if (rawUrls.length > 0) {
-          searchCache.setSearchResult(keyword, source, cacheKey, rawUrls);
+          searchCache.setSearchResult(keyword, source, options, rawUrls);
         }
         metrics.recordCacheMiss();
       }
@@ -153,9 +152,8 @@ export class Orchestrator {
       // 统一搜索数量策略：与 Link 模式一致
       const searchCount = Math.max(count * 3, 10);
       
-      // 尝试从缓存获取（使用统一的缓存键策略）
-      const cacheKey = { keyword, source, size: options.size, aspect: options.aspect };
-      const cachedUrls = searchCache.getSearchResult(keyword, source, cacheKey);
+      // 尝试从缓存获取（options 已包含 size, aspect, safeSearch）
+      const cachedUrls = searchCache.getSearchResult(keyword, source, options);
       let rawUrls;
       
       if (cachedUrls && cachedUrls.length >= count) {
@@ -166,7 +164,7 @@ export class Orchestrator {
         logger.info(`[SEARCH] "${keyword}" (target: ${searchCount})...`);
         rawUrls = await scraper.search(keyword, searchCount, options);
         if (rawUrls.length > 0) {
-          searchCache.setSearchResult(keyword, source, cacheKey, rawUrls);
+          searchCache.setSearchResult(keyword, source, options, rawUrls);
         }
         metrics.recordCacheMiss();
       }

package/src/services/orchestrator.simple.js

@@ -0,0 +1,259 @@
+/**
+ * 编排器 - 简化版
+ * 模仿主流 MCP 的实现方式：无状态、无全局缓存、每次请求独立
+ */
+
+import pLimit from 'p-limit';
+import { getScraper } from '../providers/index.js';
+import { LinkValidator } from './linkValidator.js';
+import { FileManager } from './fileManager.js';
+import { ImageProcessor } from './imageProcessor.js';
+import config from '../config/index.js';
+
+export class Orchestrator {
+  constructor() {
+    // 每个实例独立的限制器，避免全局状态
+    this.keywordLimit = pLimit(2);
+    this.linkValidator = new LinkValidator();
+    this.fileManager = new FileManager();
+    this.imageProcessor = new ImageProcessor();
+  }
+
+  /**
+   * 解析关键词字符串
+   */
+  parseKeywords(query) {
+    if (!query || typeof query !== 'string') return [];
+    return query.split(',').map(k => k.trim()).filter(k => k.length > 0);
+  }
+
+  /**
+   * 执行任务
+   */
+  async execute(params) {
+    const { 
+      query, mode, count = 10, source = 'bing', 
+      size = 'all', safeSearch = 'moderate', aspect = 'all', 
+      targetSize = null, fit = 'cover', position = 'center' 
+    } = params;
+    
+    const options = { size, safeSearch, aspect, targetSize, fit, position };
+    const startTime = Date.now();
+    const keywords = this.parseKeywords(query);
+
+    if (keywords.length === 0) {
+      return { success: false, error: '请提供有效的搜索关键词' };
+    }
+
+    // 根据模式选择处理函数
+    const processFunc = mode === 'link' 
+      ? this._processLink.bind(this)
+      : this._processDownload.bind(this);
+
+    // 并发处理关键词（使用实例级限制器）
+    const results = await Promise.all(
+      keywords.map(keyword => 
+        this.keywordLimit(() => processFunc(keyword, count, source, options))
+      )
+    );
+
+    const successResults = results.filter(r => r.success);
+    const failedResults = results.filter(r => !r.success);
+
+    return {
+      success: true,
+      mode,
+      source,
+      totalKeywords: keywords.length,
+      successCount: successResults.length,
+      failedCount: failedResults.length,
+      results,
+      duration: Date.now() - startTime,
+    };
+  }
+
+  /**
+   * 处理 Link 模式
+   */
+  async _processLink(keyword, count, source, options) {
+    const startTime = Date.now();
+    
+    try {
+      const scraper = getScraper(source);
+      const searchCount = Math.max(count * 2, 10);
+      
+      // 直接搜索，不使用缓存（简化设计）
+      const rawUrls = await scraper.search(keyword, searchCount, options);
+      
+      if (rawUrls.length === 0) {
+        return {
+          keyword,
+          success: false,
+          error: '未找到任何图片',
+          duration: Date.now() - startTime,
+        };
+      }
+
+      // 快速模式：直接返回搜索结果
+      const resultUrls = rawUrls.slice(0, count);
+
+      return {
+        keyword,
+        success: true,
+        mode: 'link',
+        totalSearched: rawUrls.length,
+        urls: resultUrls,
+        count: resultUrls.length,
+        fastMode: true,
+        duration: Date.now() - startTime,
+      };
+    } catch (error) {
+      return {
+        keyword,
+        success: false,
+        error: error.message,
+        duration: Date.now() - startTime,
+      };
+    }
+  }
+
+  /**
+   * 处理 Download 模式
+   */
+  async _processDownload(keyword, count, source, options) {
+    const startTime = Date.now();
+    
+    try {
+      const scraper = getScraper(source);
+      const searchCount = Math.max(count * 2, 10);
+      
+      // 搜索
+      const rawUrls = await scraper.search(keyword, searchCount, options);
+      
+      if (rawUrls.length === 0) {
+        return {
+          keyword,
+          success: false,
+          error: '未找到任何图片',
+          duration: Date.now() - startTime,
+        };
+      }
+
+      // 下载图片
+      const { success, failed } = await this.fileManager.downloadMany(rawUrls, keyword);
+      let resultDownloads = success.slice(0, count);
+
+      // 如果指定了目标尺寸，进行后处理
+      let processedCount = 0;
+      let processFailedCount = 0;
+      if (options.targetSize && resultDownloads.length > 0) {
+        const targetSize = this.imageProcessor.parseTargetSize(options.targetSize);
+        if (targetSize) {
+          const processResult = await this.imageProcessor.processMany(resultDownloads, {
+            width: targetSize.width,
+            height: targetSize.height,
+            fit: options.fit || 'cover',
+            position: options.position || 'center',
+          });
+          resultDownloads = processResult.success;
+          processedCount = processResult.success.length;
+          processFailedCount = processResult.failed.length;
+        }
+      }
+
+      // 保存元数据
+      let metadataPath = null;
+      if (resultDownloads.length > 0) {
+        metadataPath = await this.fileManager.saveMetadata(keyword, resultDownloads);
+      }
+
+      return {
+        keyword,
+        success: true,
+        mode: 'download',
+        totalSearched: rawUrls.length,
+        totalDownloaded: success.length,
+        totalFailed: failed.length,
+        totalProcessed: processedCount,
+        totalProcessFailed: processFailedCount,
+        files: resultDownloads,
+        count: resultDownloads.length,
+        saveDir: this.fileManager.getKeywordDir(keyword),
+        metadataPath,
+        targetSize: options.targetSize || null,
+        duration: Date.now() - startTime,
+      };
+    } catch (error) {
+      return {
+        keyword,
+        success: false,
+        error: error.message,
+        duration: Date.now() - startTime,
+      };
+    }
+  }
+
+  /**
+   * 格式化输出为 Markdown
+   */
+  formatResult(result) {
+    if (!result.success) {
+      return `## ❌ 任务失败\n\n**错误原因**: ${result.error}`;
+    }
+
+    const lines = [];
+    lines.push(`# 📷 图片抓取报告`);
+    lines.push('');
+    lines.push(`- **模式**: ${result.mode === 'link' ? '链接提取' : '本地下载'}`);
+    lines.push(`- **搜索源**: ${result.source}`);
+    lines.push(`- **关键词数量**: ${result.totalKeywords}`);
+    lines.push(`- **成功**: ${result.successCount} | **失败**: ${result.failedCount}`);
+    lines.push(`- **总耗时**: ${(result.duration / 1000).toFixed(2)}秒`);
+    lines.push('');
+
+    for (const r of result.results) {
+      lines.push(`## 🔍 关键词: ${r.keyword}`);
+      lines.push('');
+
+      if (!r.success) {
+        lines.push(`❌ **失败**: ${r.error}`);
+        lines.push('');
+        continue;
+      }
+
+      if (r.mode === 'link') {
+        lines.push(`- 搜索到: ${r.totalSearched || 0} 张`);
+        lines.push(`- 返回: ${r.count || 0} 张`);
+        lines.push(`- 耗时: ${(r.duration / 1000).toFixed(2)}秒`);
+        lines.push('');
+        lines.push('### 有效链接');
+        lines.push('');
+        (r.urls || []).forEach((url, i) => {
+          lines.push(`${i + 1}. ${url}`);
+        });
+      } else {
+        lines.push(`- 搜索到: ${r.totalSearched} 张`);
+        lines.push(`- 下载成功: ${r.totalDownloaded} 张`);
+        lines.push(`- 下载失败: ${r.totalFailed} 张`);
+        if (r.targetSize) {
+          lines.push(`- 尺寸处理: ${r.totalProcessed} 成功, ${r.totalProcessFailed} 失败`);
+          lines.push(`- 目标尺寸: ${r.targetSize}`);
+        }
+        lines.push(`- 最终保存: ${r.count} 张`);
+        lines.push(`- 存储目录: \`${r.saveDir}\``);
+        lines.push(`- 耗时: ${(r.duration / 1000).toFixed(2)}秒`);
+        lines.push('');
+        lines.push('### 已下载文件');
+        lines.push('');
+        r.files.forEach((file, i) => {
+          lines.push(`${i + 1}. \`${file.path}\``);
+        });
+      }
+      lines.push('');
+    }
+
+    return lines.join('\n');
+  }
+}
+
+export default Orchestrator;