web-fetcher-mcp 1.0.0

package/dist/utils/page-content.js

@@ -0,0 +1,139 @@
+// 提取页面内容
+export async function extractPageContent(page, contentSelector = 'body') {
+    const content = await page.evaluate((selector) => {
+        let detailElement = document.querySelector(selector);
+        let text = '';
+        let images = [];
+        // 提取表单元素信息
+        const formElements = [];
+        const allInputs = document.querySelectorAll('input, button, textarea, select, [role="button"]');
+        allInputs.forEach((el, index) => {
+            const element = el;
+            const tagName = element.tagName.toLowerCase();
+            const inputType = element.getAttribute('type') || '';
+            const id = element.id || '';
+            const className = element.className || '';
+            const name = element.getAttribute('name') || '';
+            const placeholder = element.getAttribute('placeholder') || '';
+            const textContent = element.textContent?.trim() || '';
+            // 生成推荐的选择器
+            let selectorStr = '';
+            if (id) {
+                selectorStr = `#${id}`;
+            }
+            else if (name) {
+                selectorStr = `${tagName}[name="${name}"]`;
+            }
+            else if (className) {
+                const classes = className.split(' ').filter(c => c).join('.');
+                selectorStr = `.${classes}`;
+            }
+            else {
+                selectorStr = `${tagName}:nth-of-type(${index + 1})`;
+            }
+            // 只保留重要的表单元素
+            const isImportant = tagName === 'button' ||
+                tagName === 'input' && ['text', 'password', 'email', 'tel', 'checkbox', 'submit', 'button'].includes(inputType) ||
+                tagName === 'textarea' ||
+                tagName === 'select' ||
+                element.getAttribute('role') === 'button' ||
+                /登录|Login|提交|Submit|同意|Agree|注册|Register/i.test(textContent);
+            if (isImportant) {
+                formElements.push({
+                    tagName,
+                    type: inputType || undefined,
+                    name: name || undefined,
+                    id: id || undefined,
+                    className: className || undefined,
+                    placeholder: placeholder || undefined,
+                    value: element.value || textContent || undefined,
+                    selector: selectorStr
+                });
+            }
+        });
+        if (detailElement) {
+            // 提取文本内容
+            text = detailElement.innerText || '';
+            // 提取图片
+            const imgElements = detailElement.querySelectorAll('img');
+            imgElements.forEach((img) => {
+                const src = img.getAttribute('src') || img.getAttribute('data-src');
+                if (src) {
+                    images.push(src);
+                }
+            });
+        }
+        else {
+            // 如果找不到指定元素，获取整个 HTML
+            text = document.documentElement.innerText || '';
+            // 提取整个页面的图片
+            const imgElements = document.querySelectorAll('img');
+            imgElements.forEach((img) => {
+                const src = img.getAttribute('src') || img.getAttribute('data-src');
+                if (src) {
+                    images.push(src);
+                }
+            });
+        }
+        // 生成 HTML 骨架（简化版）
+        const htmlSkeleton = (() => {
+            const body = document.body;
+            if (!body)
+                return '';
+            // 递归生成骨架
+            const generateSkeleton = (element, depth = 0) => {
+                if (depth > 3)
+                    return ''; // 限制深度
+                const tagName = element.tagName.toLowerCase();
+                const id = element.id ? ` id="${element.id}"` : '';
+                const className = element.className && typeof element.className === 'string'
+                    ? ` class="${element.className.split(' ').slice(0, 2).join(' ')}"`
+                    : '';
+                // 只保留重要标签
+                const importantTags = ['form', 'input', 'button', 'a', 'div', 'span', 'section', 'article', 'header', 'footer', 'nav'];
+                if (!importantTags.includes(tagName)) {
+                    return '';
+                }
+                // 跳过脚本和样式
+                if (['script', 'style', 'link', 'meta'].includes(tagName)) {
+                    return '';
+                }
+                let skeleton = '  '.repeat(depth) + `<${tagName}${id}${className}>`;
+                // 如果是表单元素，添加更多信息
+                if (tagName === 'input') {
+                    const type = element.getAttribute('type') || 'text';
+                    const name = element.getAttribute('name') || '';
+                    const placeholder = element.getAttribute('placeholder') || '';
+                    skeleton += ` [type=${type}${name ? ` name=${name}` : ''}${placeholder ? ` placeholder=${placeholder.slice(0, 20)}` : ''}]`;
+                }
+                if (tagName === 'button' || (tagName === 'div' && element.getAttribute('role') === 'button')) {
+                    const text = element.textContent?.trim().slice(0, 30) || '';
+                    if (text)
+                        skeleton += ` [text="${text}"]`;
+                }
+                const children = Array.from(element.children);
+                if (children.length > 0 && depth < 3) {
+                    skeleton += '\n';
+                    children.slice(0, 5).forEach(child => {
+                        const childSkeleton = generateSkeleton(child, depth + 1);
+                        if (childSkeleton)
+                            skeleton += childSkeleton + '\n';
+                    });
+                    skeleton += '  '.repeat(depth);
+                }
+                skeleton += `</${tagName}>`;
+                return skeleton;
+            };
+            return generateSkeleton(body, 0);
+        })();
+        return {
+            text,
+            images,
+            downloadedImages: [],
+            formElements: formElements.slice(0, 20), // 限制数量
+            htmlSkeleton: htmlSkeleton.slice(0, 5000) // 限制长度
+        };
+    }, contentSelector);
+    return content;
+}
+//# sourceMappingURL=page-content.js.map

package/dist/utils/page-content.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-content.js","sourceRoot":"","sources":["../../src/utils/page-content.ts"],"names":[],"mappings":"AAeA,SAAS;AACT,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAU,EAAE,kBAA0B,MAAM;IACnF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC/C,IAAI,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAuB,CAAC;QAC3E,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,MAAM,GAAa,EAAE,CAAC;QAE1B,WAAW;QACX,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CAAC,kDAAkD,CAAC,CAAC;QAEhG,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;YAC9B,MAAM,OAAO,GAAG,EAAiB,CAAC;YAClC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;YAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YACrD,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;YAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAChD,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAC9D,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;YAEtD,WAAW;YACX,IAAI,WAAW,GAAG,EAAE,CAAC;YACrB,IAAI,EAAE,EAAE,CAAC;gBACP,WAAW,GAAG,IAAI,EAAE,EAAE,CAAC;YACzB,CAAC;iBAAM,IAAI,IAAI,EAAE,CAAC;gBAChB,WAAW,GAAG,GAAG,OAAO,UAAU,IAAI,IAAI,CAAC;YAC7C,CAAC;iBAAM,IAAI,SAAS,EAAE,CAAC;gBACrB,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC9D,WAAW,GAAG,IAAI,OAAO,EAAE,CAAC;YAC9B,CAAC;iBAAM,CAAC;gBACN,WAAW,GAAG,GAAG,OAAO,gBAAgB,KAAK,GAAG,CAAC,GAAG,CAAC;YACvD,CAAC;YAED,aAAa;YACb,MAAM,WAAW,GACf,OAAO,KAAK,QAAQ;gBACpB,OAAO,KAAK,OAAO,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAC/G,OAAO,KAAK,UAAU;gBACtB,OAAO,KAAK,QAAQ;gBACpB,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,QAAQ;gBACzC,0CAA0C,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAE/D,IAAI,WAAW,EAAE,CAAC;gBAChB,YAAY,CAAC,IAAI,CAAC;oBAChB,OAAO;oBACP,IAAI,EAAE,SAAS,IAAI,SAAS;oBAC5B,IAAI,EAAE,IAAI,IAAI,SAAS;oBACvB,EAAE,EAAE,EAAE,IAAI,SAAS;oBACnB,SAAS,EAAE,SAAS,IAAI,SAAS;oBACjC,WAAW,EAAE,WAAW,IAAI,SAAS;oBACrC,KAAK,EAAG,OAA4B,CAAC,KAAK,IAAI,WAAW,IAAI,SAAS;oBACtE,QAAQ,EAAE,WAAW;iBACtB,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,IAAI,aAAa,EAAE,CAAC;YAClB,SAAS;YACT,IAAI,GAAG,aAAa,CAAC,SAAS,IAAI,EAAE,CAAC;YAErC,OAAO;YACP,MAAM,WAAW,GAAG,aAAa,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC1D,WAAW,CAAC,OAAO,CAAC,CAAC,GAAqB,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;gBACpE,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,sBAAsB;YACtB,IAAI,GAAG,QAAQ,CAAC,eAAe,CAAC,SAAS,IAAI,EAAE,CAAC;YAEhD,YAAY;YACZ,MAAM,WAAW,GAAG,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACrD,WAAW,CAAC,OAAO,CAAC,CAAC,GAAqB,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;gBACpE,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,kBAAkB;QAClB,MAAM,YAAY,GAAG,CAAC,GAAG,EAAE;YACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;YAC3B,IAAI,CAAC,IAAI;gBAAE,OAAO,EAAE,CAAC;YAErB,SAAS;YACT,MAAM,gBAAgB,GAAG,CAAC,OAAgB,EAAE,QAAgB,CAAC,EAAU,EAAE;gBACvE,IAAI,KAAK,GAAG,CAAC;oBAAE,OAAO,EAAE,CAAC,CAAC,OAAO;gBAEjC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAC9C,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,OAAO,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACnD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,OAAO,CAAC,SAAS,KAAK,QAAQ;oBAC1E,CAAC,CAAC,WAAW,OAAO,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG;oBAClE,CAAC,CAAC,EAAE,CAAC;gBAEP,UAAU;gBACV,MAAM,aAAa,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;gBACvH,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBACrC,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAED,UAAU;gBACV,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC1D,OAAO,EAAE,CAAC;gBACZ,CAAC;gBAED,IAAI,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,OAAO,GAAG,EAAE,GAAG,SAAS,GAAG,CAAC;gBAEpE,iBAAiB;gBACjB,IAAI,OAAO,KAAK,OAAO,EAAE,CAAC;oBACxB,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC;oBACpD,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;oBAChD,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;oBAC9D,QAAQ,IAAI,UAAU,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,WAAW,CAAC,CAAC,CAAC,gBAAgB,WAAW,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC;gBAC9H,CAAC;gBAED,IAAI,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO,KAAK,KAAK,IAAI,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,QAAQ,CAAC,EAAE,CAAC;oBAC7F,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;oBAC5D,IAAI,IAAI;wBAAE,QAAQ,IAAI,WAAW,IAAI,IAAI,CAAC;gBAC5C,CAAC;gBAED,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAC9C,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;oBACrC,QAAQ,IAAI,IAAI,CAAC;oBACjB,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE;wBACnC,MAAM,aAAa,GAAG,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC;wBACzD,IAAI,aAAa;4BAAE,QAAQ,IAAI,aAAa,GAAG,IAAI,CAAC;oBACtD,CAAC,CAAC,CAAC;oBACH,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACjC,CAAC;gBAED,QAAQ,IAAI,KAAK,OAAO,GAAG,CAAC;gBAC5B,OAAO,QAAQ,CAAC;YAClB,CAAC,CAAC;YAEF,OAAO,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACnC,CAAC,CAAC,EAAE,CAAC;QAEL,OAAO;YACL,IAAI;YACJ,MAAM;YACN,gBAAgB,EAAE,EAAE;YACpB,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,OAAO;YAChD,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,OAAO;SAClD,CAAC;IACJ,CAAC,EAAE,eAAe,CAAC,CAAC;IAEpB,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "web-fetcher-mcp",
+  "version": "1.0.0",
+  "description": "MCP Server for fetching any web page content with login support",
+  "main": "dist/index.js",
+  "bin": {
+    "web-fetcher-mcp": "./dist/index.js"
+  },
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "dev": "tsx src/index.ts",
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "prepublishOnly": "npm run build",
+    "release": "node publish.js",
+    "release:minor": "node publish.js minor",
+    "release:major": "node publish.js major"
+  },
+  "keywords": [
+    "mcp",
+    "web",
+    "fetcher",
+    "scraper",
+    "puppeteer"
+  ],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "@modelcontextprotocol/sdk": "^1.27.1",
+    "axios": "^1.13.6",
+    "cheerio": "^1.2.0",
+    "puppeteer": "^20.0.0",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  },
+  "volta": {
+    "node": "18.18.0"
+  }
+}