smart-image-scraper-mcp 2.8.1 → 2.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "smart-image-scraper-mcp",
-  "version": "2.8.1",
+  "version": "2.9.0",
   "description": "全网智能图片抓取 MCP 服务器 - 支持 Bing/Google 图片搜索、验证和下载",
   "main": "src/index.js",
   "type": "module",

package/src/providers/bingScraper.js

@@ -26,12 +26,31 @@ export class BingScraper extends BaseScraper {
     this.options = options;
     const pageSize = 35;
     
-    // 只获取第一页，避免触发速率限制
-    logger.info(`[Bing] Searching "${keyword}" - single page mode`);
+    // 计算需要获取的页数（最多3页，避免触发速率限制）
+    const pagesNeeded = Math.min(Math.ceil(limit / pageSize), 3);
+    logger.info(`[Bing] Searching "${keyword}" - ${pagesNeeded} page(s) for ${limit} images`);
 
     try {
-      const urls = await this._fetchPage(keyword, 0);
-      const result = urls.slice(0, limit);
+      let allUrls = [];
+      
+      // 顺序获取多页（避免并发触发限制）
+      for (let page = 0; page < pagesNeeded; page++) {
+        const offset = page * pageSize;
+        const urls = await this._fetchPage(keyword, offset);
+        allUrls = allUrls.concat(urls);
+        
+        // 如果已经够了就停止
+        if (allUrls.length >= limit) {
+          break;
+        }
+        
+        // 页间延迟，避免触发速率限制
+        if (page < pagesNeeded - 1) {
+          await this._delay(200);
+        }
+      }
+      
+      const result = allUrls.slice(0, limit);
       logger.info(`[Bing] Complete: ${result.length} URLs for "${keyword}"`);
       return result;
     } catch (error) {

package/src/providers/googleScraper.js

@@ -24,13 +24,32 @@ export class GoogleScraper extends BaseScraper {
    */
   async search(keyword, limit = 10, options = {}) {
     this.options = options;
+    const pageSize = 20; // Google 每页约20张
     
-    // 只获取第一页，避免触发速率限制
-    logger.info(`[Google] Searching "${keyword}" - single page mode`);
+    // 计算需要获取的页数（最多3页，避免触发速率限制）
+    const pagesNeeded = Math.min(Math.ceil(limit / pageSize), 3);
+    logger.info(`[Google] Searching "${keyword}" - ${pagesNeeded} page(s) for ${limit} images`);
 
     try {
-      const urls = await this._fetchPage(keyword, 0);
-      const result = urls.slice(0, limit);
+      let allUrls = [];
+      
+      // 顺序获取多页
+      for (let page = 0; page < pagesNeeded; page++) {
+        const start = page * pageSize;
+        const urls = await this._fetchPage(keyword, start);
+        allUrls = allUrls.concat(urls);
+        
+        if (allUrls.length >= limit) {
+          break;
+        }
+        
+        // 页间延迟
+        if (page < pagesNeeded - 1) {
+          await this._delay(300);
+        }
+      }
+      
+      const result = allUrls.slice(0, limit);
       logger.info(`[Google] Complete: ${result.length} URLs for "${keyword}"`);
       return result;
     } catch (error) {
@@ -38,6 +57,10 @@ export class GoogleScraper extends BaseScraper {
       return [];
     }
   }
+  
+  _delay(ms) {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
 
   /**
    * 获取单页结果