Jarvis-Brain 0.1.11.2__tar.gz → 0.1.13.9__tar.gz

{jarvis_brain-0.1.11.2 → jarvis_brain-0.1.13.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Jarvis_Brain
-Version: 0.1.11.2
+Version: 0.1.13.9
 Summary: Jarvis brain mcp
 Requires-Python: >=3.10
 Requires-Dist: beautifulsoup4

{jarvis_brain-0.1.11.2 → jarvis_brain-0.1.13.9}/mcp_tools/dp_tools.py

@@ -7,6 +7,7 @@ import os
 import time
 from typing import Any
 
+from DrissionPage.errors import NoRectError
 from fastmcp import FastMCP
 
 from tools.browser_manager import BrowserManager
@@ -162,13 +163,13 @@ def register_check_selector(mcp: FastMCP, browser_manager):
             attr_output = json.dumps(ele_attr_list, ensure_ascii=False)
         # 对attr_output逐个截断，截断的长度为：一轮最大token除以元素个数+3个点+两个引号和逗号
         return dp_mcp_message_pack(
-            f"已完成tab页:【{tab_id}】对：【{css_selector}】的检查",
+            f"已完成tab页:【{tab_id}】对：【{css_selector}】的检查，当前选中了 {len(target_eles)} 个元素",
             tab_id=tab_id,
             selector=css_selector,
             selector_ele_exist=exist_flag,
             page_size=page_size,
             offset=offset,
-            attr_output=attr_output
+            attr_output=attr_output,
         )
 
 
@@ -236,27 +237,28 @@ def register_assert_waf(mcp: FastMCP, browser_manager):
 
 
 def register_click_action(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="click_action", description="尝试点击tab页中的元素，返回元素是否可以被点击，以及是否点击成功。")
-    async def click_action(browser_port: int, tab_id: str, css_selector: str) -> dict[str, Any]:
+    @mcp.tool(name="click_action",
+              description="尝试点击tab页中的元素，返回元素是否可以被点击，以及是否点击成功。"
+                          "其中target_element_index默认为0，当传入的Selector可以定位到多个元素时，需要传入target_element_index指定具体点击目标 ")
+    async def click_action(browser_port: int, tab_id: str, css_selector: str, target_element_index: int = 0) -> dict[
+        str, Any]:
         _browser = browser_manager.get_browser(browser_port)
         target_tab = _browser.get_tab(tab_id)
         css_selector = css_selector
         if "css:" not in css_selector:
             css_selector = "css:" + css_selector
         target_eles = target_tab.eles(css_selector)
-        click_success = False
-        element_clickable = False
-        if len(target_eles) == 1:
-            target_element = target_eles[0]
-            element_clickable = target_element.states.is_clickable
-            try:
-                target_element.click()
-                click_success = True
-            except Exception as e:
-                click_success = False
-            message = f"tab页:【{tab_id}】点击【{css_selector}】 {'成功' if click_success else '失败'} 了"
+        target_element = target_eles[target_element_index]
+        element_clickable = target_element.states.is_clickable
+        try:
+            click_success = target_element.click()
+            click_success = click_success is not False
+        except Exception as e:
+            click_success = False
+        if target_element_index > 0:
+            message = f"tab页:【{tab_id}】点击【{css_selector}】【index={target_element_index}】的元素 {'成功' if click_success else '失败'} 了"
         else:
-            message = f"tab页:【{tab_id}】传入的css_selector找到了{len(target_eles)}个元素，请确保传入的css_selector可以找到唯一的一个元素"
+            message = f"tab页:【{tab_id}】点击【{css_selector}】 {'成功' if click_success else '失败'} 了"
         return dp_mcp_message_pack(
             message=message,
             browser_port=browser_port,
@@ -282,23 +284,28 @@ def register_scroll_action(mcp: FastMCP, browser_manager):
         if forward == "down":
             if pixel is None:
                 target_tab.scroll.to_half()
-            target_tab.scroll.down(pixel)
+            else:
+                target_tab.scroll.down(pixel)
         elif forward == "up":
             if pixel is None:
                 target_tab.scroll.to_top()
-            target_tab.scroll.up(pixel)
+            else:
+                target_tab.scroll.up(pixel)
         elif forward == "left":
             if pixel is None:
                 target_tab.scroll.to_leftmost()
-            target_tab.scroll.left(pixel)
+            else:
+                target_tab.scroll.left(pixel)
         elif forward == "right":
             if pixel is None:
                 target_tab.scroll.to_rightmost()
-            target_tab.scroll.right(pixel)
+            else:
+                target_tab.scroll.right(pixel)
         else:
             if pixel is None:
                 target_tab.scroll.to_half()
-            target_tab.scroll.down()
+            else:
+                target_tab.scroll.down()
         message = f"已完成对tab页:【{tab_id}】forward={forward} 的滑动"
         return dp_mcp_message_pack(
             message=message,
@@ -307,14 +314,128 @@ def register_scroll_action(mcp: FastMCP, browser_manager):
         )
 
 
+def register_mouse_hover(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="mouse_hover", description="将鼠标悬停在元素上【这个功能使用了Drissionpage的action行为链功能】")
+    async def mouse_hover(browser_port: int, tab_id: str, css_selector: str, target_element_index: int = 0) -> dict[
+        str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        target_tab = _browser.get_tab(tab_id)
+        css_selector = css_selector
+        if "css:" not in css_selector:
+            css_selector = "css:" + css_selector
+        target_eles = target_tab.eles(css_selector)
+        target_element = target_eles[target_element_index]
+        try:
+            target_tab.actions.move_to(target_element)
+            target_element.hover()
+            hover_success = True
+        except Exception as e:
+            hover_success = False
+        if target_element_index > 0:
+            message = f"tab页:【{tab_id}】hover【{css_selector}】【index={target_element_index}】的元素 {'成功' if hover_success else '失败'} 了"
+        else:
+            message = f"tab页:【{tab_id}】hover【{css_selector}】 {'成功' if hover_success else '失败'} 了"
+        # else:
+        #     message = f"tab页:【{tab_id}】传入的css_selector找到了{len(target_eles)}个元素，请确保传入的css_selector可以找到唯一的一个元素"
+        return dp_mcp_message_pack(
+            message=message,
+            browser_port=browser_port,
+            tab_id=tab_id,
+            css_selector=css_selector,
+            hoversuccess=hover_success,
+            extra_message="hover成功，页面可能有更新，请重新获取页面html，并重新分析页面Selector" if hover_success else ""
+        )
+
+
+def register_get_ele_info(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="get_ele_info", description="返回元素有关一系列的信息。"
+                                               "参数说明："
+                                               "browser_port：浏览器端口号。"
+                                               "tab_id：tab页的id。"
+                                               "css_selector：目标选择器"
+                                               "element_index：如果选择器定位到的是一个列表【或者可以定位到多个元素】，则该值用于定位这个列表中具体元素，默认值为0【列表中的第一个元素】"
+                                               "返回值说明："
+                                               "element_tag：此属性返回元素的标签名。"
+                                               "element_attrs_key：此属性以list的形式返回元素所有属性的key。"
+                                               "element_rect_size：此属性以元组形式返回元素的大小【如果元素没有位置及大小，则返回空元组】。"
+                                               "is_in_viewport：此属性以布尔值方式返回元素是否在视口中，以元素可以接受点击的点为判断。"
+                                               "is_whole_in_viewport：此属性以布尔值方式返回元素是否整个在视口中。"
+                                               "is_alive：此属性以布尔值形式返回当前元素是否仍可用。用于判断是否因页面刷新而导致元素失效。"
+                                               "is_checked：此属性以布尔值返回表单单选或多选元素是否选中。"
+                                               "is_selected：此属性以布尔值返回<select>元素中的项是否选中。"
+                                               "is_enabled：此属性以布尔值返回元素是否可用。"
+                                               "is_displayed：此属性以布尔值返回元素是否可见。"
+                                               "is_covered：此属性返回元素是否被其它元素覆盖/遮挡。如被覆盖/遮挡，返回覆盖元素的 id【无障碍树的id，无法用于Selector】，否则返回False。"
+                                               "is_clickable：此属性返回元素是否可被模拟点击，从是否有大小、是否可用、是否显示、是否响应点击判断，不判断是否被遮挡。"
+                                               "has_rect：此属性返回元素是否拥有大小和位置信息，有则返回四个角在页面上的坐标组成的列表，没有则返回False。")
+    async def get_ele_info(browser_port: int, tab_id: str, css_selector: str, element_index: int = 0) -> dict[
+        str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        target_tab = _browser.get_tab(tab_id)
+        css_selector = css_selector
+        if "css:" not in css_selector:
+            css_selector = "css:" + css_selector
+        target_eles = target_tab.eles(css_selector)
+        try:
+            target_element = target_eles[element_index]
+        except IndexError:
+            return dp_mcp_message_pack(
+                message="报错：IndexError: list index out of range。请检查Selector中是否包含了如：first-child、nth-child等字段，如果有则去掉。必须使用element_index来控制元素的选择",
+                browser_port=browser_port,
+                tab_id=tab_id,
+                css_selector=css_selector,
+                element_index=element_index,
+            )
+        try:
+            has_rect = target_element.states.has_rect,
+            element_rect_size = tuple()
+            if not has_rect:
+                element_rect_size = target_element.rect.size,
+            try:
+                child_count=target_element.child_count
+            except Exception:
+                child_count=0
+            return dp_mcp_message_pack(
+                message="元素可以被正常的选择到，以下是元素相关的一系列信息",
+                browser_port=browser_port,
+                tab_id=tab_id,
+                css_selector=css_selector,
+                element_index=element_index,
+                element_tag=target_element.tag,
+                element_attrs_key=list(target_element.attrs.keys()),
+                element_child_count=child_count,
+                element_rect_size=element_rect_size,
+                is_in_viewport=target_element.states.is_in_viewport,
+                is_whole_in_viewport=target_element.states.is_whole_in_viewport,
+                is_alive=target_element.states.is_alive,
+                is_checked=target_element.states.is_checked,
+                is_selected=target_element.states.is_selected,
+                is_enabled=target_element.states.is_enabled,
+                is_displayed=target_element.states.is_displayed,
+                is_covered=target_element.states.is_covered,
+                is_clickable=target_element.states.is_clickable,
+                has_rect=has_rect
+            )
+        except NoRectError:
+            return dp_mcp_message_pack(
+                message="报错：NoRectError: 该元素没有位置及大小。你传入的css_selector和element_index选出的元素没有rect，请确认该元素是否可见",
+                browser_port=browser_port,
+                tab_id=tab_id,
+                css_selector=css_selector,
+                element_index=element_index,
+            )
+
 def register_get_screenshot(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="get_tab_screenshot", description="尝试对传入tab页进行截图，并将截图压缩为1M大小png图片，会返回截图保存路径")
+    @mcp.tool(name="get_tab_screenshot",
+              description="尝试对传入tab页进行截图，并将截图压缩为1M大小png图片，会返回截图保存路径")
     async def get_tab_screenshot(browser_port: int, tab_id: str) -> dict[str, Any]:
         _browser = browser_manager.get_browser(browser_port)
         target_tab = _browser.get_tab(tab_id)
+        target_tab.wait.doc_loaded()
         if not os.path.exists(html_source_code_local_save_path):
             os.makedirs(html_source_code_local_save_path)
         timestamp = int(time.time() * 1000)
+        time.sleep(3)
         origin_png = target_tab.get_screenshot(as_bytes="png")
         compress_png = compress_image_bytes(origin_png)
         image_path = os.path.join(html_source_code_local_save_path, f"{browser_port}_{tab_id}_{timestamp}.png")

{jarvis_brain-0.1.11.2 → jarvis_brain-0.1.13.9}/mcp_tools/main.py

@@ -24,6 +24,8 @@ if "TeamNode-Dp" in enabled_modules:
     # 页面交互
     register_click_action(mcp, browser_manager)
     register_scroll_action(mcp, browser_manager)
+    register_mouse_hover(mcp, browser_manager)
+    register_get_ele_info(mcp, browser_manager)
 
 if "JarvisNode" in enabled_modules:
     register_assert_waf(mcp, browser_manager)

{jarvis_brain-0.1.11.2 → jarvis_brain-0.1.13.9}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "Jarvis_Brain"  # 别人下载时用的名字，必须在 PyPI 上唯一
-version = "0.1.11.2"
+version = "0.1.13.9"
 description = "Jarvis brain mcp"
 dependencies = [
     "fastmcp",

{jarvis_brain-0.1.11.2 → jarvis_brain-0.1.13.9}/tools/browser_proxy.py

@@ -140,7 +140,10 @@ def check_data_packet(packet: DataPacket, client: DPProxyClient):
         data = packet.request.postData
     domain = urlparse(url).netloc
     body = packet.response.body
-    body_str = json.dumps(body, ensure_ascii=False, separators=(',', ':'))
+    if isinstance(body, dict):
+        body_str = json.dumps(body, ensure_ascii=False, separators=(',', ':'))
+    else:
+        body_str = str(body)
     body_str_list = [body_str[i:i + one_turn_max_token] for i in range(0, len(body_str), one_turn_max_token)]
     body_completed = True
     packet_filter = client.packet_filter
@@ -155,9 +158,9 @@ def check_data_packet(packet: DataPacket, client: DPProxyClient):
             continue
         if (index + 1) != len(body_str_list):
             body_completed = False
-        if packet.response:
+        try:
             response_headers = packet.response.headers
-        else:
+        except TypeError:
             response_headers = {}
         temp_dict = {
             "url": url,
@@ -171,59 +174,4 @@ def check_data_packet(packet: DataPacket, client: DPProxyClient):
         client.packet_queue.append(temp_dict)
 
 
-def check_data_packet(packet: DataPacket, client: DPProxyClient):
-    """
-    封装监听到的数据包，并将其存放在client的packet_queue中
-    :param packet:
-    :param client:
-    :return:
-    """
-    url = packet.url
-    method = packet.request.method
-    data = None
-    if packet.request.hasPostData:
-        data = packet.request.postData
-    domain = urlparse(url).netloc
-    body = packet.response.body
-    body_str = json.dumps(body, ensure_ascii=False, separators=(',', ':'))
-    body_str_list = [body_str[i:i + one_turn_max_token] for i in range(0, len(body_str), one_turn_max_token)]
-    body_completed = True
-    packet_filter = client.packet_filter
-    domain_filter = packet_filter.get("domain_filter", None)
-    method_filter = packet_filter.get("method_filter", ["GET", "POST"])
-    for index, body_str in enumerate(body_str_list):
-        # 如果给了domain_filter并且domain没有在domain_filter中时跳过该数据包
-        if domain_filter and domain not in domain_filter:
-            continue
-        # 如果method没有在method_filter中，则跳过该数据包
-        if method not in method_filter:
-            continue
-        if (index + 1) != len(body_str_list):
-            body_completed = False
-        temp_dict = {
-            "url": url,
-            "body_completed": body_completed,
-            "method": method,
-            "request_data": data,
-            "request_headers": dict(packet.request.headers),
-            "response_headers": dict(packet.response.headers),
-            "response_body_segment": body_str.replace("\\", ""),
-        }
-        client.packet_queue.append(temp_dict)
-
-
 client_manager = DPProxyClientManager()
-
-# if __name__ == '__main__':
-#     co = ChromiumOptions().set_user_agent(
-#         "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36")
-#     tab = ChromiumPage(co).latest_tab
-#     client = DPProxyClient(tab, self_kill=False)
-#     # client = CaptchaClient(tab, self_kill=True)
-#     tab = client.get_driver(True)
-#     url = "https://api.toutiaoapi.com/feoffline/hotspot_and_local/html/hot_list/index.html?client_extra_params=%7B%22custom_log_pb%22%3A%22%7B%5C%22style_id%5C%22%3A%5C%2240030%5C%22%2C%5C%22entrance_hotspot%5C%22%3A%5C%22search%5C%22%2C%5C%22location%5C%22%3A%5C%22hot_board%5C%22%2C%5C%22category_name%5C%22%3A%5C%22hotboard_light%5C%22%7D%22%7D&count=50&log_pb=%7B%22style_id%22%3A%2240030%22%2C%22entrance_hotspot%22%3A%22search%22%2C%22location%22%3A%22hot_board%22%2C%22category_name%22%3A%22hotboard_light%22%7D&only_hot_list=1&tab_name=stream&enter_keyword=%23%E7%BE%8E%E5%9B%BD%E9%80%80%E5%87%BA66%E4%B8%AA%E5%9B%BD%E9%99%85%E7%BB%84%E7%BB%87%23"
-#     tab.get(url)
-#     for _ in range(5056):
-#         new_packet = client.pop_first_packet()
-#         print(new_packet, "23")
-#         time.sleep(1)

jarvis_brain-0.1.13.9/tools/tools.py

@@ -0,0 +1,365 @@
+import time
+import random
+import os
+import minify_html
+from DrissionPage import ChromiumPage, ChromiumOptions
+from bs4 import BeautifulSoup
+from curl_cffi import requests
+from lxml import html, etree
+import base64
+from PIL import Image
+import io
+
+compress_html_js1 = """
+function getSimplifiedDOM(node) {
+    // 1. 处理文本节点
+    if (node.nodeType === Node.TEXT_NODE) {
+        const text = node.textContent.trim();
+        return text ? text.slice(0, 100) + (text.length > 100 ? '...' : '') : null;
+    }
+
+    // 2. 过滤无用标签
+    const ignoreTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'IFRAME', 'SVG', 'LINK', 'META'];
+    if (ignoreTags.includes(node.tagName)) return null;
+    if (node.nodeType !== Node.ELEMENT_NODE) return null;
+
+    // 3. 过滤不可见元素
+    // 【注意】这里声明了第一次 style
+    const style = window.getComputedStyle(node);
+    
+    if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return null;
+    
+    // 过滤宽高太小的元素（往往是埋点空像素）
+    const rect = node.getBoundingClientRect();
+    
+    // 【修复点】删除了这里重复的 const style = ... 代码
+    // 直接使用上面已经定义好的 style 变量即可
+    
+    // 如果宽高为0，但溢出可见，说明可能有定位的子元素显示在外面
+    if ((rect.width === 0 || rect.height === 0) && style.overflow !== 'visible') return null;
+
+    // --- 开始构建标签字符串 ---
+    const tagName = node.tagName.toLowerCase();
+    let tagStr = tagName;
+
+    // A. 基础标识符 (ID 和 Class)
+    if (node.id) tagStr += `#${node.id}`;
+    if (node.className && typeof node.className === 'string') {
+        const classes = node.className.trim().split(/\s+/);
+        if (classes.length > 0) tagStr += `.${classes.join('.')}`;
+    }
+
+    // B. 关键属性白名单
+    const props = [];
+
+    // 通用重要属性
+    if (node.getAttribute('role')) props.push(`role="${node.getAttribute('role')}"`);
+    if (node.getAttribute('aria-label')) props.push(`aria-label="${node.getAttribute('aria-label')}"`);
+    if (node.getAttribute('title')) props.push(`title="${node.getAttribute('title')}"`);
+    // 建议增加这个，很多弹窗用这个属性
+    if (node.getAttribute('aria-modal')) props.push(`aria-modal="${node.getAttribute('aria-modal')}"`);
+
+    // 特定标签的特定属性
+    if (tagName === 'a') {
+        const href = node.getAttribute('href');
+        if (href && !href.startsWith('javascript')) props.push(`href="${href}"`);
+    } else if (tagName === 'input' || tagName === 'textarea' || tagName === 'select') {
+        if (node.getAttribute('type')) props.push(`type="${node.getAttribute('type')}"`);
+        if (node.getAttribute('name')) props.push(`name="${node.getAttribute('name')}"`);
+        if (node.getAttribute('placeholder')) props.push(`placeholder="${node.getAttribute('placeholder')}"`);
+        if (node.disabled) props.push('disabled');
+        if (node.checked) props.push('checked');
+    } else if (tagName === 'button') {
+        if (node.getAttribute('type')) props.push(`type="${node.getAttribute('type')}"`);
+    } else if (tagName === 'img') {
+        if (node.getAttribute('alt')) props.push(`alt="${node.getAttribute('alt')}"`);
+    } else if (tagName === 'dialog') {
+        // 保留 open 属性
+        if (node.open) props.push('open');
+    }
+
+    if (props.length > 0) {
+        tagStr += ` ${props.join(' ')}`;
+    }
+
+    // 4. 递归子节点 (包含 Shadow DOM 处理)
+    let childNodes = Array.from(node.childNodes);
+    if (node.shadowRoot) {
+        childNodes = [...childNodes, ...Array.from(node.shadowRoot.childNodes)];
+    }
+    
+    const children = childNodes
+        .map(getSimplifiedDOM)
+        .filter(n => n !== null);
+    
+    // 5. 组装输出
+    if (children.length === 0) {
+        return `<${tagStr} />`;
+    }
+    return `<${tagStr}>${children.join('')}</${tagName}>`; 
+}
+
+return getSimplifiedDOM(document.body);
+"""
+
+# 我自己优化后的版本，逻辑为：删除不可见元素、标签的任何属性value的长度大于20时直接删除这个属性、id和class采用简写方式：id=>#，class=>.
+compress_html_js="""
+function getSimplifiedDOM(node) {
+    // 全局配置：最大属性值长度
+    const MAX_ATTR_LEN = 40;
+
+    // 1. 处理文本节点
+    if (node.nodeType === Node.TEXT_NODE) {
+        const text = node.textContent.trim();
+        return text ? text.slice(0, 100) + (text.length > 100 ? '...' : '') : null;
+    }
+
+    // 2. 过滤无用标签
+    const ignoreTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'IFRAME', 'LINK', 'META', 'AUDIO', 'VIDEO', 'CANVAS'];
+    if (ignoreTags.includes(node.tagName)) return null;
+    if (node.nodeType !== Node.ELEMENT_NODE) return null;
+
+    // 3. 过滤不可见元素
+    const style = window.getComputedStyle(node);
+    if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return null;
+    
+    const rect = node.getBoundingClientRect();
+    if ((rect.width === 0 || rect.height === 0) && style.overflow !== 'visible') return null;
+
+    // --- 开始构建标签字符串 ---
+    const tagName = node.tagName.toLowerCase();
+    let tagStr = tagName;
+
+    const id = node.id;
+    const className = node.getAttribute('class');
+
+    // A. 处理 ID 简写 (#id)
+    // 限制提高到 40
+    if (id && id.length <= MAX_ATTR_LEN) {
+        tagStr += `#${id}`;
+    }
+
+    // B. 处理 Class 简写 (.class)
+    // 限制提高到 40
+    if (className && typeof className === 'string' && className.length <= MAX_ATTR_LEN) {
+        const classes = className.trim().split(/\s+/);
+        if (classes.length > 0) {
+            tagStr += `.${classes.join('.')}`;
+        }
+    }
+
+    let propsStr = '';
+
+    // C. 处理属性
+    if (node.hasAttributes()) {
+        for (const attr of node.attributes) {
+            const name = attr.name;
+            const value = attr.value;
+
+            // 1. 跳过 ID 和 Class (已在 tagStr 处理，或因过长被丢弃)
+            if (name === 'id' || name === 'class') continue;
+
+            // 2. 黑名单：直接删除 style 和 aria-label
+            if (name === 'style' || name === 'aria-label') continue;
+
+            // 3. 特殊标签：path 标签删除所有属性
+            if (tagName === 'path') continue;
+
+            // 4. 【长度与白名单逻辑】
+            // 如果不是 src 且不是 href，同时长度又超过了 40，则删除
+            const isLinkAttr = (name === 'src' || name === 'href');
+            
+            if (!isLinkAttr && value.length > MAX_ATTR_LEN) {
+                continue; 
+            }
+
+            // 5. 拼接保留的属性
+            propsStr += ` ${name}="${value.replace(/"/g, '&quot;')}"`;
+        }
+    }
+
+    // 4. 递归子节点
+    let childNodes = Array.from(node.childNodes);
+    if (node.shadowRoot) {
+        childNodes = [...childNodes, ...Array.from(node.shadowRoot.childNodes)];
+    }
+    
+    const children = childNodes
+        .map(getSimplifiedDOM)
+        .filter(n => n !== null);
+    
+    // 5. 组装输出
+    if (children.length === 0) {
+        return `<${tagStr}${propsStr} />`;
+    }
+    return `<${tagStr}${propsStr}>${children.join('')}</${tagName}>`; 
+}
+
+return getSimplifiedDOM(document.body);
+"""
+
+# 使用requests获取html，用于测试是否使用了瑞数和jsl
+def requests_html(url):
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+    }
+    response = requests.get(url, headers=headers, verify=False)
+    response.encoding = "utf-8"
+    return response.text, response.status_code
+
+
+# 使用dp无头模式获取html，用于测试是否使用了其他waf，如移动waf
+def dp_headless_html(url):
+    opt = ChromiumOptions().headless(True)
+    opt.set_argument('--no-sandbox')
+    """创建新的浏览器实例"""
+    random_port = random.randint(9934, 10034)
+    custom_data_dir = os.path.join(os.path.expanduser('~'), 'DrissionPage', "userData", f"{random_port}")
+    opt.set_user_data_path(custom_data_dir)  # 设置用户数据路径
+    opt.set_local_port(random_port)
+    page = ChromiumPage(opt)
+    tab = page.latest_tab
+    tab.get(url)
+    # todo: 目前没有更好的方式，为了数据渲染完全，只能硬等【受网速波动影响比较大】
+    time.sleep(10)
+    page_html = tab.html
+    # 无头浏览器在用完之后一定要记得再page级别进行quit
+    page.quit()
+    return page_html
+
+
+# 压缩html
+def compress_html(content, only_text=False):
+    doc = html.fromstring(content)
+    # 删除 style 和 script 标签
+    for element in doc.xpath('//style | //script'):
+        element.getparent().remove(element)
+
+    # 删除 link 标签
+    for link in doc.xpath('//link[@rel="stylesheet"]'):
+        link.getparent().remove(link)
+
+    # 删除 meta 标签（新增功能）
+    for meta in doc.xpath('//meta'):
+        meta.getparent().remove(meta)
+
+    for svg in doc.xpath('//svg'):
+        # 获取 SVG 内的文本内容
+        text_content = svg.text_content()
+        # 创建一个新的文本节点替换 SVG
+        parent = svg.getparent()
+        if parent is not None:
+            parent.text = (parent.text or '') + text_content
+            parent.remove(svg)
+
+    # 删除 style 属性
+    for element in doc.xpath('//*[@style]'):
+        element.attrib.pop('style')
+
+    # 删除所有 on* 事件属性
+    for element in doc.xpath('//*'):
+        for attr in list(element.attrib.keys()):
+            if attr.startswith('on'):
+                element.attrib.pop(attr)
+
+    result = etree.tostring(doc, encoding='unicode')
+    result = minify_html.minify(result)
+    compress_rate = round(len(content) / len(result) * 100)
+    print(f"html压缩比=> {compress_rate}%")
+    if not only_text:
+        return result, compress_rate
+    soup = BeautifulSoup(result, 'html.parser')
+    result = soup.get_text(strip=True)
+    return result, compress_rate
+
+
+# 通过cookie判断是否有waf，需要通过遇到的例子，不断的完善cookie判别函数
+def assert_waf_cookie(cookies: list):
+    for cookie in cookies:
+        cookie_name = cookie['name']
+        cookie_value = cookie['value']
+        if len(cookie_name) == 13 and len(cookie_value) == 88:
+            return True, "瑞数"
+        if "_jsl" in cookie_name:
+            return True, "加速乐"
+    return False, "没有waf"
+
+
+# 对dp_mcp的消息打包
+def dp_mcp_message_pack(message: str, **kwargs):
+    text_obj = {key: value for key, value in kwargs.items()}
+    text_obj.update({"message": message})
+    return {
+        "content": [{
+            "type": "text",
+            # "text": json.dumps(text_obj, ensure_ascii=False)
+            "text": text_obj
+        }]
+    }
+
+
+def btyes2Base64Img(target_byte):
+    """
+    把byte转为base64，用于传输图片
+    :param target_byte:
+    :return:
+    """
+    return "data:image/png;base64," + base64.b64encode(target_byte).decode()
+
+
+def compress_image_bytes(input_bytes, target_size_mb=1):
+    """
+    压缩图片字节数据到目标大小
+
+    参数:
+        input_bytes: 输入图片的字节数据
+        target_size_mb: 目标大小(MB)，默认1MB
+
+    返回:
+        压缩后的图片字节数据
+    """
+    target_size = target_size_mb * 1024 * 1024  # 转换为字节
+
+    # 从字节数据打开图片
+    img = Image.open(io.BytesIO(input_bytes))
+
+    # 如果是PNG或其他格式，转换为RGB
+    if img.mode in ('RGBA', 'LA', 'P'):
+        img = img.convert('RGB')
+
+    # 初始质量设置
+    quality = 95
+
+    # 先尝试压缩
+    output_buffer = io.BytesIO()
+    img.save(output_buffer, 'JPEG', quality=quality, optimize=True)
+    output_bytes = output_buffer.getvalue()
+
+    # 如果文件仍然太大，逐步降低质量
+    while len(output_bytes) > target_size and quality > 10:
+        quality -= 5
+        output_buffer = io.BytesIO()
+        img.save(output_buffer, 'JPEG', quality=quality, optimize=True)
+        output_bytes = output_buffer.getvalue()
+
+    # 如果降低质量还不够，尝试缩小尺寸
+    if len(output_bytes) > target_size:
+        width, height = img.size
+
+        while len(output_bytes) > target_size and quality > 10:
+            # 缩小10%
+            width = int(width * 0.9)
+            height = int(height * 0.9)
+            img_resized = img.resize((width, height), Image.Resampling.LANCZOS)
+            output_buffer = io.BytesIO()
+            img_resized.save(output_buffer, 'JPEG', quality=quality, optimize=True)
+            output_bytes = output_buffer.getvalue()
+
+    final_size = len(output_bytes) / (1024 * 1024)
+    # print(f"压缩完成！")
+    # print(f"原始大小: {len(input_bytes) / (1024 * 1024):.2f}MB")
+    # print(f"压缩后大小: {final_size:.2f}MB")
+    # print(f"最终质量: {quality}")
+
+    return output_bytes
+

jarvis_brain-0.1.11.2/tools/tools.py

@@ -1,267 +0,0 @@
-import time
-import random
-import os
-import minify_html
-from DrissionPage import ChromiumPage, ChromiumOptions
-from bs4 import BeautifulSoup
-from curl_cffi import requests
-from lxml import html, etree
-import base64
-from PIL import Image
-import io
-
-compress_html_js = """
-    function getSimplifiedDOM(node) {
-        // 1. 处理文本节点
-        if (node.nodeType === Node.TEXT_NODE) {
-            const text = node.textContent.trim();
-            // 限制文本长度，避免大段文章消耗 token，保留前100个字符通常足够定位
-            return text ? text.slice(0, 100) + (text.length > 100 ? '...' : '') : null;
-        }
-
-        // 2. 过滤无用标签
-        const ignoreTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'IFRAME', 'SVG', 'LINK', 'META'];
-        if (ignoreTags.includes(node.tagName)) return null;
-        if (node.nodeType !== Node.ELEMENT_NODE) return null;
-
-        // 3. 过滤不可见元素
-        const style = window.getComputedStyle(node);
-        if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return null;
-        // 过滤宽高太小的元素（往往是埋点空像素）
-        const rect = node.getBoundingClientRect();
-        if (rect.width === 0 || rect.height === 0) return null;
-
-        // --- 开始构建标签字符串 ---
-        const tagName = node.tagName.toLowerCase();
-        let tagStr = tagName;
-
-        // A. 基础标识符 (ID 和 Class)
-        if (node.id) tagStr += `#${node.id}`;
-        if (node.className && typeof node.className === 'string') {
-            // 过滤掉 Tailwind 等太长且无语义的 class，保留有意义的业务 class
-            // 这里简单处理，全部保留，让 LLM 自己判断
-            const classes = node.className.trim().split(/\s+/);
-            if (classes.length > 0) tagStr += `.${classes.join('.')}`;
-        }
-
-        // B. 关键属性白名单 (这是你指出问题的核心修复)
-        const props = [];
-
-        // 通用重要属性
-        if (node.getAttribute('role')) props.push(`role="${node.getAttribute('role')}"`);
-        if (node.getAttribute('aria-label')) props.push(`aria-label="${node.getAttribute('aria-label')}"`);
-        if (node.getAttribute('title')) props.push(`title="${node.getAttribute('title')}"`);
-
-        // 特定标签的特定属性
-        if (tagName === 'a') {
-            const href = node.getAttribute('href');
-            // 只保留有意义的链接，忽略 javascript:;
-            if (href && !href.startsWith('javascript')) props.push(`href="${href}"`);
-        } else if (tagName === 'input' || tagName === 'textarea' || tagName === 'select') {
-            if (node.getAttribute('type')) props.push(`type="${node.getAttribute('type')}"`);
-            if (node.getAttribute('name')) props.push(`name="${node.getAttribute('name')}"`);
-            if (node.getAttribute('placeholder')) props.push(`placeholder="${node.getAttribute('placeholder')}"`);
-            if (node.disabled) props.push('disabled');
-            if (node.checked) props.push('checked');
-        } else if (tagName === 'button') {
-            if (node.getAttribute('type')) props.push(`type="${node.getAttribute('type')}"`);
-        } else if (tagName === 'img') {
-            if (node.getAttribute('alt')) props.push(`alt="${node.getAttribute('alt')}"`);
-        }
-
-        if (props.length > 0) {
-            tagStr += ` ${props.join(' ')}`;
-        }
-
-        // 4. 递归子节点
-        const children = Array.from(node.childNodes)
-            .map(getSimplifiedDOM)
-            .filter(n => n !== null);
-
-        // 5. 组装输出
-        // 如果没有子节点，也没有ID/Class，也不是输入框/图片/链接，那这个标签可能只是布局用的 div，可以考虑跳过它直接返回子节点内容
-        // 但为了保持结构完整，我们暂时保留它
-        if (children.length === 0) {
-            // 自闭合标签或空标签
-            return `<${tagStr} />`;
-        }
-        return `<${tagStr}>${children.join('')}</${tagName}>`; // 结束标签只保留 tagName 节省 token
-    }
-
-    return getSimplifiedDOM(document.body);
-"""
-
-
-# 使用requests获取html，用于测试是否使用了瑞数和jsl
-def requests_html(url):
-    headers = {
-        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
-    }
-    response = requests.get(url, headers=headers, verify=False)
-    response.encoding = "utf-8"
-    return response.text, response.status_code
-
-
-# 使用dp无头模式获取html，用于测试是否使用了其他waf，如移动waf
-def dp_headless_html(url):
-    opt = ChromiumOptions().headless(True)
-    opt.set_argument('--no-sandbox')
-    """创建新的浏览器实例"""
-    random_port = random.randint(9934, 10034)
-    custom_data_dir = os.path.join(os.path.expanduser('~'), 'DrissionPage', "userData", f"{random_port}")
-    opt.set_user_data_path(custom_data_dir)  # 设置用户数据路径
-    opt.set_local_port(random_port)
-    page = ChromiumPage(opt)
-    tab = page.latest_tab
-    tab.get(url)
-    # todo: 目前没有更好的方式，为了数据渲染完全，只能硬等【受网速波动影响比较大】
-    time.sleep(10)
-    page_html = tab.html
-    # 无头浏览器在用完之后一定要记得再page级别进行quit
-    page.quit()
-    return page_html
-
-
-# 压缩html
-def compress_html(content, only_text=False):
-    doc = html.fromstring(content)
-    # 删除 style 和 script 标签
-    for element in doc.xpath('//style | //script'):
-        element.getparent().remove(element)
-
-    # 删除 link 标签
-    for link in doc.xpath('//link[@rel="stylesheet"]'):
-        link.getparent().remove(link)
-
-    # 删除 meta 标签（新增功能）
-    for meta in doc.xpath('//meta'):
-        meta.getparent().remove(meta)
-
-    for svg in doc.xpath('//svg'):
-        # 获取 SVG 内的文本内容
-        text_content = svg.text_content()
-        # 创建一个新的文本节点替换 SVG
-        parent = svg.getparent()
-        if parent is not None:
-            parent.text = (parent.text or '') + text_content
-            parent.remove(svg)
-
-    # 删除 style 属性
-    for element in doc.xpath('//*[@style]'):
-        element.attrib.pop('style')
-
-    # 删除所有 on* 事件属性
-    for element in doc.xpath('//*'):
-        for attr in list(element.attrib.keys()):
-            if attr.startswith('on'):
-                element.attrib.pop(attr)
-
-    result = etree.tostring(doc, encoding='unicode')
-    result = minify_html.minify(result)
-    compress_rate = round(len(content) / len(result) * 100)
-    print(f"html压缩比=> {compress_rate}%")
-    if not only_text:
-        return result, compress_rate
-    soup = BeautifulSoup(result, 'html.parser')
-    result = soup.get_text(strip=True)
-    return result, compress_rate
-
-
-# 通过cookie判断是否有waf，需要通过遇到的例子，不断的完善cookie判别函数
-def assert_waf_cookie(cookies: list):
-    for cookie in cookies:
-        cookie_name = cookie['name']
-        cookie_value = cookie['value']
-        if len(cookie_name) == 13 and len(cookie_value) == 88:
-            return True, "瑞数"
-        if "_jsl" in cookie_name:
-            return True, "加速乐"
-    return False, "没有waf"
-
-
-# 对dp_mcp的消息打包
-def dp_mcp_message_pack(message: str, **kwargs):
-    text_obj = {key: value for key, value in kwargs.items()}
-    text_obj.update({"message": message})
-    return {
-        "content": [{
-            "type": "text",
-            # "text": json.dumps(text_obj, ensure_ascii=False)
-            "text": text_obj
-        }]
-    }
-
-
-def btyes2Base64Img(target_byte):
-    """
-    把byte转为base64，用于传输图片
-    :param target_byte:
-    :return:
-    """
-    return "data:image/png;base64," + base64.b64encode(target_byte).decode()
-
-
-def compress_image_bytes(input_bytes, target_size_mb=1):
-    """
-    压缩图片字节数据到目标大小
-
-    参数:
-        input_bytes: 输入图片的字节数据
-        target_size_mb: 目标大小(MB)，默认1MB
-
-    返回:
-        压缩后的图片字节数据
-    """
-    target_size = target_size_mb * 1024 * 1024  # 转换为字节
-
-    # 从字节数据打开图片
-    img = Image.open(io.BytesIO(input_bytes))
-
-    # 如果是PNG或其他格式，转换为RGB
-    if img.mode in ('RGBA', 'LA', 'P'):
-        img = img.convert('RGB')
-
-    # 初始质量设置
-    quality = 95
-
-    # 先尝试压缩
-    output_buffer = io.BytesIO()
-    img.save(output_buffer, 'JPEG', quality=quality, optimize=True)
-    output_bytes = output_buffer.getvalue()
-
-    # 如果文件仍然太大，逐步降低质量
-    while len(output_bytes) > target_size and quality > 10:
-        quality -= 5
-        output_buffer = io.BytesIO()
-        img.save(output_buffer, 'JPEG', quality=quality, optimize=True)
-        output_bytes = output_buffer.getvalue()
-
-    # 如果降低质量还不够，尝试缩小尺寸
-    if len(output_bytes) > target_size:
-        width, height = img.size
-
-        while len(output_bytes) > target_size and quality > 10:
-            # 缩小10%
-            width = int(width * 0.9)
-            height = int(height * 0.9)
-            img_resized = img.resize((width, height), Image.Resampling.LANCZOS)
-            output_buffer = io.BytesIO()
-            img_resized.save(output_buffer, 'JPEG', quality=quality, optimize=True)
-            output_bytes = output_buffer.getvalue()
-
-    final_size = len(output_bytes) / (1024 * 1024)
-    # print(f"压缩完成！")
-    # print(f"原始大小: {len(input_bytes) / (1024 * 1024):.2f}MB")
-    # print(f"压缩后大小: {final_size:.2f}MB")
-    # print(f"最终质量: {quality}")
-
-    return output_bytes
-
-# todo: 大致盘一下各种判定的逻辑【以下的所有压缩比之间的差距均取“绝对值”】
-#  1. 如果requests、无头、有头获取到的压缩比之间从差距都在15%以内，则认定该页面是静态页面，此时优先使用requests请求
-#  2. 如果requests的status_code为特定的412，或者521，则判定是瑞数和jsl。[此时还有一个特点：requests的压缩比会与其他两种方式获取到的压缩比差距非常大(一两千的那种)]
-#  3. 如果requests、无头、有头获取到的压缩比之间差距都在40%以上，则判定该页面只可以用有头采集
-#  4. 如果无头和有头获取到的压缩比之间差距小于15%，但是requests和无头的差距大于40%，则认定该页面可以使用无头浏览器采集
-#  5. 如果requests和有头获取到的压缩比之间差距小于15%，但是无头和有头的差距大于40%，则认定该页面优先使用有头浏览器采集
-#  【此时可能是：1.使用了别的检测无头的waf。2.网站使用瑞数，但是这次请求没有拦截requests（不知道是不是瑞数那边故意设置的），
-#   此时如果想进一步判定是否是瑞数，可以使用有头浏览器取一下cookies，如果cookies里面存在瑞数的cookie，那么就可以断定是瑞数】