smart-image-scraper-mcp 2.12.3 → 2.13.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smart-image-scraper-mcp",
-  "version": "2.12.3",
+  "version": "2.13.0",
   "description": "全网智能图片抓取 MCP 服务器 - 支持 Bing/Google 图片搜索、验证和下载",
   "main": "src/index.js",
   "type": "module",

package/src/config/index.js

@@ -33,7 +33,7 @@ function ensureSaveRoot(savePath) {
     fs.unlinkSync(testFile);
     return savePath;
   } catch (error) {
-    console.error(`[CONFIG] Cannot write to SAVE_ROOT: ${savePath}, using default`);
+    process.stderr.write(`[CONFIG] Cannot write to SAVE_ROOT: ${savePath}, using default\n`);
     const defaultPath = path.join(__dirname, '../../images');
     if (!fs.existsSync(defaultPath)) {
       fs.mkdirSync(defaultPath, { recursive: true });

package/src/index.js

@@ -46,12 +46,16 @@ const SMART_SCRAPER_TOOL = {
 【核心功能】
 1. 搜索图片链接 (mode=link) - 返回验证过的图片URL列表
 2. 下载图片 (mode=download) - 下载到本地，自动按质量排序优先高清
-3. 尺寸统一 (targetSize) - 下载后自动裁剪/缩放到指定尺寸
-4. 宽高比过滤 (aspect) - 横向/竖向/正方形
+3. 搜索+下载 (mode=both) - 同时返回链接列表和下载文件，适合需要链接备份的场景
+4. 尺寸统一 (targetSize) - 下载后自动裁剪/缩放到指定尺寸
+5. 宽高比过滤 (aspect) - 横向/竖向/正方形
+6. 自定义下载路径 (savePath) - 指定图片保存位置，不填则保存到MCP服务器项目下images目录
 
 【参数选择指南】
 - 用户要"找/搜索/查找图片" → mode="link"
 - 用户要"下载/保存/获取图片" → mode="download"
+- 用户要"搜索并下载/链接和下载都要" → mode="both"
+- 用户要"保存到指定目录" → savePath="D:/my/path"
 - 用户要"高清/大图/壁纸" → size="large" 或 "wallpaper"，quality="high"
 - 用户要"高质量/精选/优质" → quality="high"
 - 用户要"电脑壁纸/横屏/横向" → aspect="wide"
@@ -75,7 +79,9 @@ const SMART_SCRAPER_TOOL = {
 3. 下载电脑壁纸并统一为1080p: {"query":"风景","mode":"download","count":10,"aspect":"wide","targetSize":"desktop_1080p"}
 4. 下载手机壁纸: {"query":"动漫","mode":"download","count":10,"aspect":"tall","targetSize":"mobile_hd"}
 5. 批量下载多类图片: {"query":"猫,狗,兔子","mode":"download","count":5}
-6. 获取高质量图片: {"query":"风景","mode":"link","count":5,"size":"large","quality":"high"}`,
+6. 获取高质量图片: {"query":"风景","mode":"link","count":5,"size":"large","quality":"high"}
+7. 搜索并下载到指定目录: {"query":"风景","mode":"both","count":5,"savePath":"D:/photos"}
+8. 下载到自定义目录: {"query":"猫","mode":"download","count":5,"savePath":"D:/my/images"}`,
   inputSchema: {
     type: 'object',
     properties: {
@@ -85,8 +91,8 @@ const SMART_SCRAPER_TOOL = {
       },
       mode: {
         type: 'string',
-        enum: ['link', 'download'],
-        description: "运行模式。link=仅返回验证过的图片URL列表（用户只需要链接时使用）；download=下载图片到本地文件系统（用户说下载/保存时使用）",
+        enum: ['link', 'download', 'both'],
+        description: "运行模式。link=仅返回验证过的图片URL列表；download=下载图片到本地；both=同时返回链接和下载文件（用户需要链接备份时使用）",
       },
       count: {
         type: 'number',
@@ -139,6 +145,15 @@ const SMART_SCRAPER_TOOL = {
         description: '最小文件大小过滤。文件越大通常质量越高。any=不限制；建议高清图片用100kb以上',
         default: 'any',
       },
+      savePath: {
+        type: 'string',
+        description: '自定义图片保存路径（绝对路径）。仅对download和both模式有效。不填则默认保存到MCP服务器项目下images目录。示例: "D:/my/photos" 或 "C:/Users/xxx/Pictures"',
+      },
+      filterHotlink: {
+        type: 'boolean',
+        description: '是否过滤防盗链图片。默认true（开启过滤）。设为false可获取更多结果但部分链接可能无法直接在浏览器打开',
+        default: true,
+      },
     },
     required: ['query', 'mode'],
   },
@@ -168,9 +183,9 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
     };
   }
 
-  if (!args.mode || !['link', 'download'].includes(args.mode)) {
+  if (!args.mode || !['link', 'download', 'both'].includes(args.mode)) {
     return {
-      content: [{ type: 'text', text: "错误: 请指定有效的运行模式 (mode): 'link' 或 'download'" }],
+      content: [{ type: 'text', text: "错误: 请指定有效的运行模式 (mode): 'link', 'download' 或 'both'" }],
       isError: true,
     };
   }
@@ -196,6 +211,8 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
       safeSearch: args.safeSearch || 'moderate',
       quality: ['fast', 'balanced', 'high'].includes(args.quality) ? args.quality : 'balanced',
       minFileSize: ['any', '50kb', '100kb', '200kb', '500kb', '1mb'].includes(args.minFileSize) ? args.minFileSize : 'any',
+      savePath: args.savePath && typeof args.savePath === 'string' ? args.savePath.trim() : null,
+      filterHotlink: args.filterHotlink !== false,  // 默认 true
     };
 
     // 使用 Promise.race 确保一定会在超时内返回
@@ -229,7 +246,7 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
   } catch (error) {
     clearTimeout(mcpTimeoutId);
     orchestrator.cleanup();
-    console.error(`[MCP Error] ${error.message}`);
+    process.stderr.write(`[MCP Error] ${error.message}\n`);
     return {
       content: [{ 
         type: 'text', 
@@ -242,16 +259,16 @@ server.setRequestHandler(CallToolRequestSchema, async (request) => {
 
 // 启动服务器（主流做法：最简启动，使用 stderr 输出日志避免干扰 stdio 通信）
 async function main() {
-  console.error(`[MCP] Starting Smart Image Scraper v${packageJson.version}`);
-  console.error(`[MCP] Save root: ${config.SAVE_ROOT}`);
+  process.stderr.write(`[MCP] Starting Smart Image Scraper v${packageJson.version}\n`);
+  process.stderr.write(`[MCP] Save root: ${config.SAVE_ROOT}\n`);
   
   const transport = new StdioServerTransport();
   await server.connect(transport);
   
-  console.error('[MCP] Server is running');
+  process.stderr.write('[MCP] Server is running\n');
 }
 
 main().catch((error) => {
-  console.error(`[MCP] Startup error: ${error.message}`);
+  process.stderr.write(`[MCP] Startup error: ${error.message}\n`);
   process.exit(1);
 });

package/src/infrastructure/logger.js

@@ -43,6 +43,10 @@ class Logger {
     this.logFile = options.logFile || process.env.LOG_FILE;
     this.maxLogSize = options.maxLogSize || 10 * 1024 * 1024; // 10MB
     
+    // stderr 限流计数器（每秒窗口）
+    this._stderrWindowStart = 0;
+    this._stderrWindowCount = 0;
+    
     // 初始化文件日志
     if (this.logFile) {
       this._initFileLogging();
@@ -127,17 +131,26 @@ class Logger {
   }
 
   /**
-   * 写入日志
+   * 写入日志（带限流保护）
    */
   _write(level, message, data) {
     if (level < this.level) return;
 
     const formatted = this._format(level, message, data);
     
-    // 输出到 stderr（使用 process.stderr.write 避免 console.error 阻塞 MCP stdio）
-    // 仅输出 WARN 及以上级别到 stderr，减少 IO 压力
+    // 输出到 stderr（带限流：每秒最多 20 条，防止高频调用时 stderr 缓冲区堆积阻塞事件循环）
     if (level >= LogLevel.WARN) {
-      process.stderr.write(formatted + '\n');
+      const now = Date.now();
+      if (now - this._stderrWindowStart > 1000) {
+        // 新的1秒窗口
+        this._stderrWindowStart = now;
+        this._stderrWindowCount = 0;
+      }
+      if (this._stderrWindowCount < 20) {
+        this._stderrWindowCount++;
+        process.stderr.write(formatted + '\n');
+      }
+      // 超出限流的日志静默丢弃，不阻塞
     }
     
     // 输出到文件（异步写入，避免阻塞事件循环）

package/src/services/fileManager.js

@@ -15,11 +15,21 @@ import config from '../config/index.js';
 const globalDownloadLimit = pLimit(config.MAX_DOWNLOAD_CONCURRENCY || 10);
 
 export class FileManager {
-  constructor() {
-    this.saveRoot = config.SAVE_ROOT;
+  constructor(options = {}) {
+    this.saveRoot = options.saveRoot || config.SAVE_ROOT;
     this.limit = globalDownloadLimit; // 使用全局共享限制器
   }
 
+  /**
+   * 设置自定义保存根目录
+   * @param {string} savePath - 绝对路径
+   */
+  setSaveRoot(savePath) {
+    if (savePath && typeof savePath === 'string') {
+      this.saveRoot = savePath;
+    }
+  }
+
   /**
    * 清理文件名，移除非法字符
    * @param {string} filename - 原始文件名

package/src/services/orchestrator.js

@@ -174,11 +174,16 @@ export class Orchestrator {
       // 检查是否已中止
       if (signal?.aborted) throw new Error('操作已取消');
 
-      // link 模式：过滤掉有防盗链保护的 URL（用户无法直接在浏览器中打开）
-      const filteredRawUrls = rawUrls.filter(url => !this.linkValidator.isHotlinkProtected(url));
-      const hotlinkCount = rawUrls.length - filteredRawUrls.length;
-      if (hotlinkCount > 0) {
-        logger.warn(`[HOTLINK] "${keyword}" - filtered ${hotlinkCount} hotlink-protected URLs`);
+      // 根据 filterHotlink 参数决定是否过滤防盗链 URL
+      const shouldFilterHotlink = options.filterHotlink !== false;
+      let filteredRawUrls = rawUrls;
+      let hotlinkCount = 0;
+      if (shouldFilterHotlink) {
+        filteredRawUrls = rawUrls.filter(url => !this.linkValidator.isHotlinkProtected(url));
+        hotlinkCount = rawUrls.length - filteredRawUrls.length;
+        if (hotlinkCount > 0) {
+          logger.warn(`[HOTLINK] "${keyword}" - filtered ${hotlinkCount} hotlink-protected URLs`);
+        }
       }
 
       if (fastMode) {
@@ -394,6 +399,189 @@ export class Orchestrator {
     }
   }
 
+  /**
+   * 处理单个关键词 - Both 模式（同时返回链接和下载文件）
+   * @param {string} keyword - 关键词
+   * @param {number} count - 需要的图片数量
+   * @param {string} source - 搜索源
+   * @param {Object} options - 搜索选项
+   * @returns {Promise<Object>} - 处理结果
+   */
+  async processKeywordBoth(keyword, count, source, options = {}) {
+    const startTime = Date.now();
+    const KEYWORD_TIMEOUT = 45000; // both 模式45秒超时（包含下载）
+    
+    // 检查是否已被中止
+    if (this.abortController?.signal?.aborted) {
+      return { keyword, success: false, error: '操作已取消', duration: 0 };
+    }
+    
+    let timeoutId;
+    const timeoutPromise = new Promise((_, reject) => {
+      timeoutId = setTimeout(() => reject(new Error(`关键词 "${keyword}" 处理超时(45秒)`)), KEYWORD_TIMEOUT);
+    });
+    
+    try {
+      const result = await Promise.race([
+        this._processKeywordBothInternal(keyword, count, source, options, startTime),
+        timeoutPromise
+      ]);
+      clearTimeout(timeoutId);
+      return result;
+    } catch (error) {
+      clearTimeout(timeoutId);
+      logger.error(`Process keyword both error: ${keyword}`, { error: error.message });
+      return {
+        keyword,
+        success: false,
+        error: error.message,
+        duration: Date.now() - startTime,
+      };
+    }
+  }
+
+  async _processKeywordBothInternal(keyword, count, source, options, startTime) {
+    const qualityMode = options.quality || 'balanced';
+    const fastMode = qualityMode === 'fast';
+    const prioritizeQuality = qualityMode === 'high';
+    const minFileSize = this._parseMinFileSize(options.minFileSize);
+    const signal = this.abortController?.signal;
+    const shouldFilterHotlink = options.filterHotlink !== false;
+    
+    try {
+      const scraper = getScraper(source);
+      const searchCount = Math.max(count * 3, 10);
+      
+      // 搜索
+      const cachedUrls = searchCache.getSearchResult(keyword, source, options);
+      let rawUrls;
+      
+      if (cachedUrls && cachedUrls.length >= count) {
+        logger.info(`[CACHE] "${keyword}" - ${cachedUrls.length} URLs`);
+        rawUrls = cachedUrls;
+        metrics.recordCacheHit();
+      } else {
+        if (signal?.aborted) throw new Error('操作已取消');
+        logger.info(`[SEARCH] "${keyword}" (target: ${searchCount})...`);
+        rawUrls = await scraper.search(keyword, searchCount, options);
+        if (rawUrls.length > 0) {
+          searchCache.setSearchResult(keyword, source, options, rawUrls);
+        }
+        metrics.recordCacheMiss();
+      }
+      
+      if (rawUrls.length === 0) {
+        return {
+          keyword,
+          success: false,
+          error: '未找到任何图片',
+          duration: Date.now() - startTime,
+        };
+      }
+
+      if (signal?.aborted) throw new Error('操作已取消');
+
+      // 防盗链过滤（用于链接列表展示）
+      let linkUrls = rawUrls;
+      let hotlinkCount = 0;
+      if (shouldFilterHotlink) {
+        linkUrls = rawUrls.filter(url => !this.linkValidator.isHotlinkProtected(url));
+        hotlinkCount = rawUrls.length - linkUrls.length;
+        if (hotlinkCount > 0) {
+          logger.warn(`[HOTLINK] "${keyword}" - filtered ${hotlinkCount} hotlink-protected URLs`);
+        }
+      }
+
+      // 验证链接
+      let validatedUrls;
+      let qualityModeLabel;
+      
+      if (fastMode) {
+        validatedUrls = linkUrls.slice(0, count);
+        qualityModeLabel = '快速模式（跳过验证）';
+      } else {
+        const maxValidate = Math.min(linkUrls.length, count * 2 + 5);
+        const urlsToValidate = linkUrls.slice(0, maxValidate);
+        const { valid } = await this.linkValidator.validateMany(urlsToValidate, {
+          fetchQuality: prioritizeQuality,
+          sortByQuality: prioritizeQuality,
+          minFileSize: minFileSize,
+          signal,
+        });
+        
+        let filteredValid = valid;
+        if (minFileSize > 0) {
+          filteredValid = valid.filter(v => {
+            const size = v.quality?.contentLength || 0;
+            return size >= minFileSize || size === 0;
+          });
+        }
+        
+        validatedUrls = filteredValid.slice(0, count).map(v => v.url);
+        qualityModeLabel = prioritizeQuality ? '高质量模式（验证+排序）' : '平衡模式（验证）';
+      }
+
+      if (signal?.aborted) throw new Error('操作已取消');
+
+      // 下载图片（使用所有原始 URL，不受防盗链过滤影响，因为下载时会加 Referer）
+      const downloadUrls = fastMode ? rawUrls.slice(0, count * 2 + 5) : validatedUrls;
+      const { success, failed } = await this.fileManager.downloadMany(downloadUrls, keyword);
+      
+      let resultDownloads = success.slice(0, count);
+
+      // 如果指定了目标尺寸，进行后处理
+      let processedCount = 0;
+      let processFailedCount = 0;
+      if (options.targetSize && resultDownloads.length > 0) {
+        const targetSize = this.imageProcessor.parseTargetSize(options.targetSize);
+        if (targetSize) {
+          logger.info(`Processing images to ${targetSize.width}x${targetSize.height}`);
+          const processResult = await this.imageProcessor.processMany(resultDownloads, {
+            width: targetSize.width,
+            height: targetSize.height,
+            fit: options.fit || 'cover',
+            position: options.position || 'center',
+          });
+          resultDownloads = processResult.success;
+          processedCount = processResult.success.length;
+          processFailedCount = processResult.failed.length;
+        }
+      }
+
+      // 保存元数据
+      let metadataPath = null;
+      if (resultDownloads.length > 0) {
+        metadataPath = await this.fileManager.saveMetadata(keyword, resultDownloads);
+      }
+
+      return {
+        keyword,
+        success: true,
+        mode: 'both',
+        totalSearched: rawUrls.length,
+        hotlinkFiltered: hotlinkCount,
+        // 链接部分
+        urls: validatedUrls,
+        urlCount: validatedUrls.length,
+        qualityMode,
+        qualityModeLabel,
+        // 下载部分
+        totalDownloaded: success.length,
+        totalFailed: failed.length,
+        totalProcessed: processedCount,
+        totalProcessFailed: processFailedCount,
+        files: resultDownloads,
+        count: resultDownloads.length,
+        saveDir: this.fileManager.getKeywordDir(keyword),
+        metadataPath,
+        targetSize: options.targetSize || null,
+        duration: Date.now() - startTime,
+      };
+    } catch (error) {
+      throw error;
+    }
+  }
+
   /**
    * 执行任务 - 直接执行（不使用队列，避免 MCP 兼容性问题）
    * @param {Object} params - 任务参数
@@ -468,8 +656,14 @@ export class Orchestrator {
    * 内部执行逻辑
    */
   async _executeInternal(params) {
-    const { query, mode, count = config.DEFAULT_COUNT, source = config.DEFAULT_SOURCE, size = 'all', safeSearch = 'moderate', aspect = 'all', targetSize = null, fit = 'cover', position = 'center', quality = 'balanced', minFileSize = 'any' } = params;
-    const options = { size, safeSearch, aspect, targetSize, fit, position, quality, minFileSize };
+    const { query, mode, count = config.DEFAULT_COUNT, source = config.DEFAULT_SOURCE, size = 'all', safeSearch = 'moderate', aspect = 'all', targetSize = null, fit = 'cover', position = 'center', quality = 'balanced', minFileSize = 'any', savePath = null, filterHotlink = true } = params;
+    const options = { size, safeSearch, aspect, targetSize, fit, position, quality, minFileSize, filterHotlink };
+    
+    // 如果指定了自定义保存路径，更新 fileManager
+    if (savePath && (mode === 'download' || mode === 'both')) {
+      this.fileManager.setSaveRoot(savePath);
+      logger.info(`[Orchestrator] Custom save path: ${savePath}`);
+    }
     
     const startTime = Date.now();
     let keywords = this.parseKeywords(query);
@@ -491,9 +685,14 @@ export class Orchestrator {
     logger.info(`Starting task: mode=${mode}, keywords=${keywords.join(', ')}, count=${count}, source=${source}`);
 
     // 根据模式选择处理函数
-    const processFunc = mode === 'link' 
-      ? this.processKeywordLink.bind(this)
-      : this.processKeywordDownload.bind(this);
+    let processFunc;
+    if (mode === 'link') {
+      processFunc = this.processKeywordLink.bind(this);
+    } else if (mode === 'both') {
+      processFunc = this.processKeywordBoth.bind(this);
+    } else {
+      processFunc = this.processKeywordDownload.bind(this);
+    }
 
     // 串行处理关键词，避免阻塞事件循环
     const results = [];
@@ -552,7 +751,8 @@ export class Orchestrator {
     const lines = [];
     lines.push(`# 📷 图片抓取报告`);
     lines.push('');
-    lines.push(`- **模式**: ${result.mode === 'link' ? '链接提取' : '本地下载'}`);
+    const modeLabels = { link: '链接提取', download: '本地下载', both: '链接提取+本地下载' };
+    lines.push(`- **模式**: ${modeLabels[result.mode] || result.mode}`);
     lines.push(`- **搜索源**: ${result.source}`);
     lines.push(`- **关键词数量**: ${result.totalKeywords}`);
     lines.push(`- **成功**: ${result.successCount} | **失败**: ${result.failedCount}`);
@@ -583,7 +783,37 @@ export class Orchestrator {
         (r.urls || []).forEach((url, i) => {
           lines.push(`${i + 1}. ${url}`);
         });
+      } else if (r.mode === 'both') {
+        // both 模式：同时显示链接和下载文件
+        lines.push(`- 搜索到: ${r.totalSearched || 0} 张`);
+        if (r.hotlinkFiltered > 0) {
+          lines.push(`- 防盗链过滤: ${r.hotlinkFiltered} 张`);
+        }
+        lines.push(`- 质量模式: ${r.qualityModeLabel || '快速模式'}`);
+        lines.push(`- 有效链接: ${r.urlCount || 0} 张`);
+        lines.push(`- 下载成功: ${r.totalDownloaded} 张`);
+        lines.push(`- 下载失败: ${r.totalFailed} 张`);
+        if (r.targetSize) {
+          lines.push(`- 尺寸处理: ${r.totalProcessed} 成功, ${r.totalProcessFailed} 失败`);
+          lines.push(`- 目标尺寸: ${r.targetSize}`);
+        }
+        lines.push(`- 最终保存: ${r.count} 张`);
+        lines.push(`- 存储目录: \`${r.saveDir}\``);
+        lines.push(`- 耗时: ${(r.duration / 1000).toFixed(2)}秒`);
+        lines.push('');
+        lines.push('### 有效链接');
+        lines.push('');
+        (r.urls || []).forEach((url, i) => {
+          lines.push(`${i + 1}. ${url}`);
+        });
+        lines.push('');
+        lines.push('### 已下载文件');
+        lines.push('');
+        (r.files || []).forEach((file, i) => {
+          lines.push(`${i + 1}. \`${file.path}\``);
+        });
       } else {
+        // download 模式
         lines.push(`- 搜索到: ${r.totalSearched} 张`);
         lines.push(`- 下载成功: ${r.totalDownloaded} 张`);
         lines.push(`- 下载失败: ${r.totalFailed} 张`);