web-fetcher-mcp 1.0.9 → 1.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +312 -98
- package/dist/index.js +550 -3
- package/dist/index.js.map +1 -1
- package/dist/tools/click-element.d.ts +1 -1
- package/dist/tools/click-element.d.ts.map +1 -1
- package/dist/tools/click-element.js +219 -4
- package/dist/tools/click-element.js.map +1 -1
- package/dist/tools/execute-js.d.ts +16 -0
- package/dist/tools/execute-js.d.ts.map +1 -0
- package/dist/tools/execute-js.js +79 -0
- package/dist/tools/execute-js.js.map +1 -0
- package/dist/tools/get-console-logs.d.ts +6 -0
- package/dist/tools/get-console-logs.d.ts.map +1 -0
- package/dist/tools/get-console-logs.js +85 -0
- package/dist/tools/get-console-logs.js.map +1 -0
- package/dist/tools/get-element-attributes.d.ts +10 -0
- package/dist/tools/get-element-attributes.d.ts.map +1 -0
- package/dist/tools/get-element-attributes.js +90 -0
- package/dist/tools/get-element-attributes.js.map +1 -0
- package/dist/tools/get-page-info.d.ts +14 -0
- package/dist/tools/get-page-info.d.ts.map +1 -0
- package/dist/tools/get-page-info.js +163 -0
- package/dist/tools/get-page-info.js.map +1 -0
- package/dist/tools/hover-element.d.ts +9 -0
- package/dist/tools/hover-element.d.ts.map +1 -0
- package/dist/tools/hover-element.js +90 -0
- package/dist/tools/hover-element.js.map +1 -0
- package/dist/tools/navigate-history.d.ts +8 -0
- package/dist/tools/navigate-history.d.ts.map +1 -0
- package/dist/tools/navigate-history.js +45 -0
- package/dist/tools/navigate-history.js.map +1 -0
- package/dist/tools/open-devtools.d.ts +18 -0
- package/dist/tools/open-devtools.d.ts.map +1 -0
- package/dist/tools/open-devtools.js +64 -0
- package/dist/tools/open-devtools.js.map +1 -0
- package/dist/tools/scroll-page.d.ts +11 -0
- package/dist/tools/scroll-page.d.ts.map +1 -0
- package/dist/tools/scroll-page.js +85 -0
- package/dist/tools/scroll-page.js.map +1 -0
- package/dist/tools/select-option.d.ts +12 -0
- package/dist/tools/select-option.d.ts.map +1 -0
- package/dist/tools/select-option.js +197 -0
- package/dist/tools/select-option.js.map +1 -0
- package/dist/tools/upload-file.d.ts +10 -0
- package/dist/tools/upload-file.d.ts.map +1 -0
- package/dist/tools/upload-file.js +99 -0
- package/dist/tools/upload-file.js.map +1 -0
- package/dist/tools/wait-for-element.d.ts +11 -0
- package/dist/tools/wait-for-element.d.ts.map +1 -0
- package/dist/tools/wait-for-element.js +68 -0
- package/dist/tools/wait-for-element.js.map +1 -0
- package/dist/tools/web-page-fetcher.js +1 -1
- package/dist/tools/web-page-fetcher.js.map +1 -1
- package/dist/types/index.d.ts +111 -0
- package/dist/types/index.d.ts.map +1 -1
- package/dist/utils/browser.d.ts.map +1 -1
- package/dist/utils/browser.js +12 -11
- package/dist/utils/browser.js.map +1 -1
- package/dist/utils/console-logger.d.ts +27 -0
- package/dist/utils/console-logger.d.ts.map +1 -0
- package/dist/utils/console-logger.js +274 -0
- package/dist/utils/console-logger.js.map +1 -0
- package/dist/utils/html-skeleton.d.ts +6 -0
- package/dist/utils/html-skeleton.d.ts.map +1 -0
- package/dist/utils/html-skeleton.js +63 -0
- package/dist/utils/html-skeleton.js.map +1 -0
- package/dist/utils/page-content.d.ts.map +1 -1
- package/dist/utils/page-content.js +9 -55
- package/dist/utils/page-content.js.map +1 -1
- package/package.json +1 -1
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
import { getHtmlSkeleton } from './html-skeleton.js';
|
|
1
2
|
// 提取页面内容
|
|
2
3
|
export async function extractPageContent(page, contentSelector = 'body', includeHtmlSkeleton = false, includeFormElements = false) {
|
|
3
|
-
const content = await page.evaluate((selector, includeSkeleton, includeFormElems) => {
|
|
4
|
+
const content = await page.evaluate((selector, includeSkeleton, includeFormElems, skeletonFuncStr) => {
|
|
5
|
+
// eslint-disable-next-line no-eval
|
|
6
|
+
const getHtmlSkeleton = eval(`(${skeletonFuncStr})`);
|
|
4
7
|
let detailElement = document.querySelector(selector);
|
|
5
8
|
let text = '';
|
|
6
9
|
let images = [];
|
|
@@ -77,65 +80,16 @@ export async function extractPageContent(page, contentSelector = 'body', include
|
|
|
77
80
|
}
|
|
78
81
|
});
|
|
79
82
|
}
|
|
80
|
-
// 生成 HTML
|
|
81
|
-
const htmlSkeleton = includeSkeleton ? (
|
|
82
|
-
const body = document.body;
|
|
83
|
-
if (!body)
|
|
84
|
-
return '';
|
|
85
|
-
// 递归生成骨架
|
|
86
|
-
const generateSkeleton = (element, depth = 0) => {
|
|
87
|
-
if (depth > 3)
|
|
88
|
-
return ''; // 限制深度
|
|
89
|
-
const tagName = element.tagName.toLowerCase();
|
|
90
|
-
const id = element.id ? ` id="${element.id}"` : '';
|
|
91
|
-
const className = element.className && typeof element.className === 'string'
|
|
92
|
-
? ` class="${element.className.split(' ').slice(0, 2).join(' ')}"`
|
|
93
|
-
: '';
|
|
94
|
-
// 只保留重要标签
|
|
95
|
-
const importantTags = ['form', 'input', 'button', 'a', 'div', 'span', 'section', 'article', 'header', 'footer', 'nav'];
|
|
96
|
-
if (!importantTags.includes(tagName)) {
|
|
97
|
-
return '';
|
|
98
|
-
}
|
|
99
|
-
// 跳过脚本和样式
|
|
100
|
-
if (['script', 'style', 'link', 'meta'].includes(tagName)) {
|
|
101
|
-
return '';
|
|
102
|
-
}
|
|
103
|
-
let skeleton = ' '.repeat(depth) + `<${tagName}${id}${className}>`;
|
|
104
|
-
// 如果是表单元素,添加更多信息
|
|
105
|
-
if (tagName === 'input') {
|
|
106
|
-
const type = element.getAttribute('type') || 'text';
|
|
107
|
-
const name = element.getAttribute('name') || '';
|
|
108
|
-
const placeholder = element.getAttribute('placeholder') || '';
|
|
109
|
-
skeleton += ` [type=${type}${name ? ` name=${name}` : ''}${placeholder ? ` placeholder=${placeholder.slice(0, 20)}` : ''}]`;
|
|
110
|
-
}
|
|
111
|
-
if (tagName === 'button' || (tagName === 'div' && element.getAttribute('role') === 'button')) {
|
|
112
|
-
const text = element.textContent?.trim().slice(0, 30) || '';
|
|
113
|
-
if (text)
|
|
114
|
-
skeleton += ` [text="${text}"]`;
|
|
115
|
-
}
|
|
116
|
-
const children = Array.from(element.children);
|
|
117
|
-
if (children.length > 0 && depth < 3) {
|
|
118
|
-
skeleton += '\n';
|
|
119
|
-
children.slice(0, 5).forEach(child => {
|
|
120
|
-
const childSkeleton = generateSkeleton(child, depth + 1);
|
|
121
|
-
if (childSkeleton)
|
|
122
|
-
skeleton += childSkeleton + '\n';
|
|
123
|
-
});
|
|
124
|
-
skeleton += ' '.repeat(depth);
|
|
125
|
-
}
|
|
126
|
-
skeleton += `</${tagName}>`;
|
|
127
|
-
return skeleton;
|
|
128
|
-
};
|
|
129
|
-
return generateSkeleton(body, 0);
|
|
130
|
-
})() : '';
|
|
83
|
+
// 生成 HTML 骨架
|
|
84
|
+
const htmlSkeleton = includeSkeleton ? getHtmlSkeleton(document.body) : '';
|
|
131
85
|
return {
|
|
132
86
|
text,
|
|
133
87
|
images,
|
|
134
88
|
downloadedImages: [],
|
|
135
|
-
formElements: formElements.slice(0,
|
|
136
|
-
htmlSkeleton: htmlSkeleton ? htmlSkeleton.slice(0,
|
|
89
|
+
formElements: formElements.slice(0, 50),
|
|
90
|
+
htmlSkeleton: htmlSkeleton ? htmlSkeleton.slice(0, 100000) : undefined
|
|
137
91
|
};
|
|
138
|
-
}, contentSelector, includeHtmlSkeleton, includeFormElements);
|
|
92
|
+
}, contentSelector, includeHtmlSkeleton, includeFormElements, getHtmlSkeleton.toString());
|
|
139
93
|
return content;
|
|
140
94
|
}
|
|
141
95
|
//# sourceMappingURL=page-content.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"page-content.js","sourceRoot":"","sources":["../../src/utils/page-content.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"page-content.js","sourceRoot":"","sources":["../../src/utils/page-content.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAcrD,SAAS;AACT,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAU,EAAE,kBAA0B,MAAM,EAAE,sBAA+B,KAAK,EAAE,sBAA+B,KAAK;IAC/J,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,EAAE,eAAe,EAAE,gBAAgB,EAAE,eAAe,EAAE,EAAE;QACnG,mCAAmC;QACnC,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,eAAe,GAAG,CAAC,CAAC;QAErD,IAAI,aAAa,GAAG,QAAQ,CAAC,aAAa,CAAC,QAAQ,CAAuB,CAAC;QAC3E,IAAI,IAAI,GAAG,EAAE,CAAC;QACd,IAAI,MAAM,GAAa,EAAE,CAAC;QAE1B,oBAAoB;QACpB,MAAM,YAAY,GAAkB,EAAE,CAAC;QACvC,IAAI,gBAAgB,EAAE,CAAC;YACrB,MAAM,SAAS,GAAG,QAAQ,CAAC,gBAAgB,CAAC,kDAAkD,CAAC,CAAC;YAEhG,SAAS,CAAC,OAAO,CAAC,CAAC,EAAE,EAAE,KAAK,EAAE,EAAE;gBAC9B,MAAM,OAAO,GAAG,EAAiB,CAAC;gBAClC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC;gBAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBACrD,MAAM,EAAE,GAAG,OAAO,CAAC,EAAE,IAAI,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,EAAE,CAAC;gBAC1C,MAAM,IAAI,GAAG,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;gBAChD,MAAM,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;gBAC9D,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;gBAEtD,WAAW;gBACX,IAAI,WAAW,GAAG,EAAE,CAAC;gBACrB,IAAI,EAAE,EAAE,CAAC;oBACP,WAAW,GAAG,IAAI,EAAE,EAAE,CAAC;gBACzB,CAAC;qBAAM,IAAI,IAAI,EAAE,CAAC;oBAChB,WAAW,GAAG,GAAG,OAAO,UAAU,IAAI,IAAI,CAAC;gBAC7C,CAAC;qBAAM,IAAI,SAAS,EAAE,CAAC;oBACrB,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;oBAC9D,WAAW,GAAG,IAAI,OAAO,EAAE,CAAC;gBAC9B,CAAC;qBAAM,CAAC;oBACN,WAAW,GAAG,GAAG,OAAO,gBAAgB,KAAK,GAAG,CAAC,GAAG,CAAC;gBACvD,CAAC;gBAED,aAAa;gBACb,MAAM,WAAW,GACf,OAAO,KAAK,QAAQ;oBACpB,OAAO,KAAK,OAAO,IAAI,CAAC,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;oBAC/G,OAAO,KAAK,UAAU;oBACtB,OAAO,KAAK,QAAQ;oBACpB,OAAO,CAAC,YAAY,CAAC,MAAM,CAAC,KAAK,QAAQ;oBACzC,0CAA0C,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAE/D,IAAI,WAAW,EAAE,CAAC;oBAChB,YAAY,CAAC,IAAI,CAAC;wBAChB,OAAO;wBACP,IAAI,EAAE,SAAS,IAAI,SAAS;wBAC5B,IAAI,EAAE,IAAI,IAAI,SAAS;wBACvB,EAAE,EAAE,EAAE,IAAI,SAAS;wBACnB,SAAS,EAAE,SAAS,IAAI,SAAS;wBACjC,WAAW,EAAE,WAAW,IAAI,SAAS;wBACrC,KAAK,EAAG,OAA4B,CAAC,KAAK,IAAI,WAAW,IAAI,SAAS;wBACtE,QAAQ,EAAE,WAAW;qBACtB,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,aAAa,EAAE,CAAC;YAClB,SAAS;YACT,IAAI,GAAG,aAAa,CAAC,SAAS,IAAI,EAAE,CAAC;YAErC,OAAO;YACP,MAAM,WAAW,GAAG,aAAa,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YAC1D,WAAW,CAAC,OAAO,CAAC,CAAC,GAAqB,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;gBACpE,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,sBAAsB;YACtB,IAAI,GAAG,QAAQ,CAAC,eAAe,CAAC,SAAS,IAAI,EAAE,CAAC;YAEhD,YAAY;YACZ,MAAM,WAAW,GAAG,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC;YACrD,WAAW,CAAC,OAAO,CAAC,CAAC,GAAqB,EAAE,EAAE;gBAC5C,MAAM,GAAG,GAAG,GAAG,CAAC,YAAY,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;gBACpE,IAAI,GAAG,EAAE,CAAC;oBACR,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;QAED,aAAa;QACb,MAAM,YAAY,GAAG,eAAe,CAAC,CAAC,CAAC,eAAe,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAE3E,OAAO;YACL,IAAI;YACJ,MAAM;YACN,gBAAgB,EAAE,EAAE;YACpB,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACvC,YAAY,EAAE,YAAY,CAAC,CAAC,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS;SACvE,CAAC;IACJ,CAAC,EAAE,eAAe,EAAE,mBAAmB,EAAE,mBAAmB,EAAE,eAAe,CAAC,QAAQ,EAAE,CAAC,CAAC;IAE1F,OAAO,OAAO,CAAC;AACjB,CAAC"}
|