Jarvis-Brain 0.1.5.5__tar.gz → 0.1.10.0__tar.gz

{jarvis_brain-0.1.5.5 → jarvis_brain-0.1.10.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Jarvis_Brain
-Version: 0.1.5.5
+Version: 0.1.10.0
 Summary: Jarvis brain mcp
 Requires-Python: >=3.10
 Requires-Dist: beautifulsoup4

{jarvis_brain-0.1.5.5 → jarvis_brain-0.1.10.0}/README.md

@@ -4,8 +4,8 @@
 
 一个基于 FastMCP 和 DrissionPage 的浏览器自动化 MCP 服务器
 
-[![Python Version](https://img.shields.io/badge/python-3.10~3.12-blue.svg)](https://www.python.org/downloads/)
-[![Version](https://img.shields.io/badge/version-0.1.4.5-green.svg)](https://github.com/yourusername/jarvis-mcp)
+[![Python Version](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/)
+[![Version](https://img.shields.io/badge/version-0.1.9.12-green.svg)](https://github.com/yourusername/jarvis-mcp)
 
 ## 📖 简介
 

jarvis_brain-0.1.10.0/mcp_tools/__init__.py

@@ -0,0 +1,5 @@
+test_str="""
+ \\"X-XSS-Protection\\": \\"1; mode=block\\", \\"Server\\": \\"-\\"}, \\"response_body_segment\\": \\"pename\\\\\\": \\\\\\"SpuItemAttribute\\\\\\"}, {\\\\\\"code\\\\\\": \\\\\\"imageVersion\\\\\\", \\\\\\"name\\\\\\": \\\\\\"imageVersion\\\\\\", \\\\\\"subAttrList\\\\\\": [{\\\\\\"attrName\\\\\\": \\\\\\"1\\\\\\", \\\\\\"attrValue\\\\\\": \\\\\\"1\\\\\\", \\\\\\"__typename\\\\\\": \\\\\\"SubAttribute\\\\\\"}], \\\\\\"__typename\\\\\\": \\\\\\"SpuItemAttribute\\\\\\"}, {\\\\\\"code\\\\\\": \\\\\\"PLPhover\\\\\\", \\\\\\"name\\\\\\": \\\\\\"PLPhover\\\\\\", \\\\\\"subAttrList\\\\\\": [{\\\\\\"attrName\\\\\\": \\\\\\"2\\\\\\", \\\\\\"attrValue\\\\\\": \\\\\\"2\\\\\\", \\\\\\"__typename\\\\\\": \\\\\\"SubAttribute\\\\\\"}], \\\\\\"__typename\\\\\\": \\\\\\"SpuItemAttribute\\\\\\"}, {\\\\\\"code\\\\\\": \\\\\\"官网\\\\\\", \\\\\\"name\\\\\\": \\\\\\"官网\\\\\\", \\\\\\"subAttrList\\\\\\": [{\\\\\\"attrName\\\\\\": \\\\\\"是\\\\\\", \\\\\\"attrValue\\\\\\": \\\\\\"true\\\\\\", \\\\\\"__typename\\\\\\": \\\\\\"SubAttribute\\\\\\"}], \\\\\\"__typename\\\\\\": \\\\\\"SpuItemAttribute\\\\\\"}, {\\\\\\"code\\\\\\": \\\\\\"小程序\\\\\\", \\\\\\"name\\\\\\": \\\\\\"小程序\\\\\\", \\\\\\"subAttrList\\\\\\": [{\\\\\\"attrName\\\\\\": \\\\\\"是\\\\\\", \\\\\\"attrValue\\\\\\": \\\\\\"true\\\\\\", \\\\\\"__typename\\\\\\": \\\\\\"SubAttribute\\\\\\"}], \\\\\\"__typename\\\\\\": \\\\\\"SpuItemAttribute\\\\\\"}], \\\\\\"__typename\\\\\\": \\\\\\"SpuAttribute\\\\\\"}, \\\\\\"labelList\\\\\\": [{\\\\\\"name\\\\\\": \\\\\\"new\\\\\\", \\\\\\"value\\\\\\": \\\\\\"新品\\\\\\", \\\\\\"excludeValue\\\\\\": null, \\\\\\"__typename\\\\\\": \\\\\\"BaseLabel\\\\\\"}], \\\\\\"__typename\\\\\\": \\\\\\"Product2\\\\\\"}, {\\\\\\"baseInfo\\\\\\": {\\\\\\"spuCode\\\\\\": \\\\\\"864428CVE0G1000\\\\\\", \\\\\\"title\\\\\\": \\\\\\"互扣式双G带扣窄版腰带\\\\\\", \\\\\\"salePrice\\\\\\": 4300.0, \\\\\\"style\\\\\\": \\\\\\"864428\\\\\\", \\\\\\"categorys\\\\\\": [{\\\\\\"navFrontName\\\\\\": \\\\\\"女士风尚-手工plp\\\\\\", \\\\\\"frontName\\\\\\": null, \\\\\\"code\\\\\\": \\\\\\"67d7d4c5eb9d2100016b4fe5\\\\\\", \\\\\\"parentCode\\\\\\": null, \\\\\\"businessCode\\\\\\": \\\\\\"women-fashion手工plp\\\\\\", \\\\\\"__typename\\\\\\": \\\\\\"Category\\\\\\"}, {\\\\\\"navFrontName\\\\\\": \\\\\\"女士-women>女士配饰-women-accessories>女士腰带-women-accessories-belts\\\\\\", \\\\\\"frontName\\\\\\": null, \\\\\\"code\\\\\\": \\\\\\"642670bf0ae2090001133cbe\\\\\\", \\\\\\"parentCode\\\\\\": null, \\\\\\"businessCode\\\\\\": \\\\\\"women-accessories-belts\\\\\\", \\\\\\"__typename\\\\\\": \\\\\\"Category\\\\\\"}, {\\\\\\"navFrontName\\\\\\": \\\\\\"女士-women>女士配饰-women-accessories\\\\\\", \\\\\\"frontName\\\\\\": null, \\\\\\"code\\\\\\": \\\\\\"642670bf0ae2090001133cbd\\\\\\", \\\\\\"parentCode\\\\\\": null, \\\\\\"businessCode\\\\\\": \\\\\\"women-accessories\\\\\\",
+"""
+decoded = test_str.encode().decode('unicode_escape')
+print(decoded)

jarvis_brain-0.1.10.0/mcp_tools/chrome_devtools_tools.py

@@ -0,0 +1,11 @@
+"""
+这个文件中提供的工具作为谷歌官方mcp：chrome-devtools-mcp的辅助工具，仅提供功能补充与浏览器实例管理
+"""
+import hashlib
+import json
+import os
+from typing import Any
+
+from fastmcp import FastMCP
+
+from tools.browser_manager import BrowserManager

jarvis_brain-0.1.10.0/mcp_tools/dp_tools.py

@@ -0,0 +1,306 @@
+"""
+这个文件中提供的工具作为独立的Drissionpage mcp工具
+"""
+import hashlib
+import json
+import os
+from typing import Any
+
+from fastmcp import FastMCP
+
+from tools.browser_manager import BrowserManager
+from tools.tools import compress_html, requests_html, dp_headless_html, assert_waf_cookie, dp_mcp_message_pack, \
+    compress_html_js
+from tools.browser_proxy import DPProxyClient, DPProxyClientManager
+
+html_source_code_local_save_path = os.path.join(os.getcwd(), "html-source-code")
+waf_status_code_dict = {
+    412: "瑞数",
+    521: "加速乐"
+}
+# 一轮最大输入，以免单个html最大长度超过ai最大输入
+one_turn_max_token = 8000
+
+
+def register_visit_url(mcp: FastMCP, browser_manager: BrowserManager, client_manager: DPProxyClientManager):
+    @mcp.tool(name="visit_url",
+              description="使用Drissionpage打开url访问某个网站，并开始监听初始tab页的所有的XHR请求"
+                          "当需要使用手机版浏览器Ua时use_mobile_user_agent为True"
+                          "如果想要以域名对packet进行过滤，可以传入想要过滤的域名列表。默认是：None。"
+                          "如果想要以method对packet进行过滤，可以传入想要过滤的method列表，默认是：['GET', 'POST']")
+    async def visit_url(url: str, domain_filter: list = None, method_filter: list = ["GET", "POST"],
+                        use_mobile_user_agent: bool = False) -> dict[str, Any]:
+        mobile_user_agent = None
+        if use_mobile_user_agent:
+            mobile_user_agent = "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36"
+        port, _browser = browser_manager.create_browser(mobile_user_agent)
+        tab = _browser.get_tab()
+        packet_filter = {
+            "domain_filter": domain_filter,
+            "method_filter": method_filter,
+        }
+        client_manager.create_client(tab, packet_filter)
+        tab.get(url)
+        tab_id = tab.tab_id
+        return dp_mcp_message_pack(
+            f"已在[{port}]端口创建浏览器对象，并已打开链接：{url}，打开的模式是：{'手机版' if use_mobile_user_agent else '电脑版'}",
+            tab_id=tab_id,
+            browser_port=port
+        )
+
+
+def register_get_new_tab(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
+    @mcp.tool(name="get_new_tab",
+              description="使用Drissionpage创建一个新的tab页，在新的tab页中打开url，并开始监听新的tab页的所有XHR请求"
+                          "如果想要以域名对packet进行过滤，可以传入想要过滤的域名列表。默认是：None。"
+                          "如果想要以method对packet进行过滤，可以传入想要过滤的method列表，默认是：['GET', 'POST']")
+    async def get_new_tab(browser_port: int, url: str, domain_filter: list = None,
+                          method_filter: list = ["GET", "POST"]) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        tab = _browser.new_tab()
+        packet_filter = {
+            "domain_filter": domain_filter,
+            "method_filter": method_filter,
+        }
+        client_manager.create_client(tab, packet_filter)
+        tab.get(url)
+        _browser.activate_tab(tab)
+        tab_id = tab.tab_id
+        return dp_mcp_message_pack(f"已创建新的tab页，并打开链接：{url}", tab_id=tab_id)
+
+
+def register_pop_first_packet(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
+    @mcp.tool(name="pop_first_packet",
+              description="每调用一次就会弹出传入的tab页所监听到的数据包中的第一个packet_message，当一个packet_message的response body过长时会被切分成多个包，具体一个请求是否还有下一个包，可以参考body_completed字段")
+    async def pop_first_packet(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        client = client_manager.get_client(tab_id)
+        current_queue_size, packet_message = client.pop_first_packet()
+        message = f"tab页:【{tab_id}】，暂时没有监听到XHR数据包"
+        if packet_message:
+            message = f"tab页:【{tab_id}】，监听到XHR数据包，当前数据包队列中还剩 {current_queue_size} 条数据，如果还剩数据为0，可以暂时稍后再次调用该方法"
+        if (packet_message is None) and current_queue_size:
+            message = f"tab页:【{tab_id}】，当前弹出的第一个数据包不符合过滤条件，当前数据包队列中还剩 {current_queue_size} 条数据，请不要改变条件，继续弹出下一个数据包"
+        return dp_mcp_message_pack(
+            message,
+            browser_port=browser_port,
+            tab_id=tab_id,
+            packet_message=packet_message,
+            current_queue_size=current_queue_size,
+        )
+
+
+def register_get_html(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="get_html", description="使用Drissionpage获取某一个tab页的html")
+    async def get_html(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        tab = _browser.get_tab(tab_id)
+        file_name_prefix = hashlib.md5(str(tab.title).encode('utf-8')).hexdigest()
+        if not os.path.exists(html_source_code_local_save_path):
+            os.makedirs(html_source_code_local_save_path)
+        # min_html, compress_rate = compress_html(tab.html)
+        min_html = tab.run_js(compress_html_js)
+        # html_str_list = [min_html[i:i + one_turn_max_token] for i in range(0, len(min_html), one_turn_max_token)]
+        html_file_list = []
+        for index, html_str in enumerate([min_html]):
+            file_name = file_name_prefix + f"_{tab_id}_segment{index}.html"
+            abs_path = os.path.join(html_source_code_local_save_path, file_name)
+            with open(abs_path, "w", encoding="utf-8") as f:
+                f.write(html_str)
+            html_file_list.append(abs_path)
+        message = f"已保存tab页：【{tab_id}】的html源码片段共{len(html_file_list)}个"
+        return dp_mcp_message_pack(message, tab_id=tab_id, htmls_local_path=html_file_list)
+
+
+def register_switch_tab(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="switch_tab", description="根据传入的tab_id切换到对应的tab页", )
+    async def switch_tab(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        _browser.activate_tab(tab_id)
+        return dp_mcp_message_pack(f"已将tab页:【{tab_id}】切换至最前端")
+
+
+def register_close_tab(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="close_tab", description="根据传入的tab_id关闭tab页", )
+    async def close_tab(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        _browser.close_tabs(tab_id)
+        return dp_mcp_message_pack(f"已将tab页:【{tab_id}】关闭")
+
+
+def register_check_selector(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="check_selector",
+              description="查找tab页中是否包含元素，并返回元素attr_name所对应的值。"
+                          "当要选择的元素包含过多元素时，需要传入offset和page_size来分批查看元素，一般不建议调整page_size，更推荐你调整offset"
+                          "同时如果单个元素属性值太长，函数会进行截断。一般的单个元素的属性值超过300个字符的就会触发截断，截断后会在最后拼接'...'")
+    async def check_selector(browser_port: int, tab_id: str, css_selector: str, attr_name: str = "text",
+                             offset: int = 0, page_size: int = 10) -> dict[
+        str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        target_tab = _browser.get_tab(tab_id)
+        css_selector = css_selector
+        if "css:" not in css_selector:
+            css_selector = "css:" + css_selector
+        target_eles = target_tab.eles(css_selector)
+        exist_flag = False
+        if len(target_eles) != 0:
+            exist_flag = True
+        if len(target_eles) > page_size:
+            target_eles = target_eles[offset:offset + page_size]
+        slice_seg = max(300, one_turn_max_token // (page_size + 6))
+        if attr_name == "text":
+            ele_attr_list = [i.text.replace("\n", "") for i in target_eles]
+            ele_attr_list = [attr_str[:slice_seg] for attr_str in ele_attr_list]
+            # 如果经过截断遍历后的字符串长度与截断长度相等，则默认截断了
+            ele_attr_list = [attr_str + "..." if len(attr_str) == slice_seg else attr_str for attr_str in ele_attr_list]
+            attr_output = "\n".join(ele_attr_list)
+        else:
+            ele_attr_list = [i.attr(attr_name) for i in target_eles]
+            ele_attr_list = [attr_str[:slice_seg] for attr_str in ele_attr_list if attr_str]
+            ele_attr_list = [attr_str + "..." if len(attr_str) == slice_seg else attr_str for attr_str in ele_attr_list]
+            attr_output = json.dumps(ele_attr_list, ensure_ascii=False)
+        # 对attr_output逐个截断，截断的长度为：一轮最大token除以元素个数+3个点+两个引号和逗号
+        return dp_mcp_message_pack(
+            f"已完成tab页:【{tab_id}】对：【{css_selector}】的检查",
+            tab_id=tab_id,
+            selector=css_selector,
+            selector_ele_exist=exist_flag,
+            page_size=page_size,
+            offset=offset,
+            attr_output=attr_output
+        )
+
+
+def register_quit_browser(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="quit_browser", description="退出浏览器会话，关闭浏览器")
+    async def quit_browser(browser_port: int) -> dict[str, Any]:
+        flag, _browser = browser_manager.remove_page(browser_port)
+        if flag:
+            _browser.quit()
+        return dp_mcp_message_pack(
+            f"浏览器[{browser_port}]，退出会话，关闭浏览器{'成功' if flag else '失败'}",
+            browser_port=browser_port,
+            quit_flag=flag
+        )
+
+
+def register_assert_waf(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="assert_waf",
+              description="通过对比requests、有头浏览器、无头浏览器获取到的html，判断网页是否使用了waf以及是否为动态渲染的网页")
+    async def assert_waf(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        target_tab = _browser.get_tab(tab_id)
+        recommend_team = "drissionpage_head"
+        head_cookies = target_tab.cookies()
+        # 通过cookie判断是否有waf
+        waf_flag, waf_type = assert_waf_cookie(head_cookies)
+        head_html = target_tab.html
+        min_head_html, head_rate = compress_html(head_html, only_text=True)
+        raw_html, status_code = requests_html(target_tab.url)
+        min_raw_html, raw_rate = compress_html(raw_html, only_text=True)
+        r_h_rate_diff = abs(head_rate - raw_rate)
+        # 如果有已知的防火墙，则不浪费时间使用无头获取html和压缩比了
+        if waf_flag or status_code in waf_status_code_dict.keys():
+            return dp_mcp_message_pack(
+                f"已完成tab页:【{tab_id}】的分析，该tab页存在waf",
+                tab_id=tab_id,
+                recommend_team=recommend_team,
+                raw_head_rate_difference=r_h_rate_diff,
+                raw_headless_rate_difference=0,
+                head_headless_rate_difference=0
+            )
+
+        headless_html = dp_headless_html(target_tab.url)
+        min_headless_html, headless_rate = compress_html(headless_html, only_text=True)
+        r_hless_rate_diff = abs(raw_rate - headless_rate)
+        h_hless_rate_diff = abs(head_rate - headless_rate)
+        # 最优情况：requests，dp有头和无头拿到的结果基本一致，认定为没有防护的静态网页
+        if r_h_rate_diff < 40 and r_hless_rate_diff < 40 and h_hless_rate_diff < 40:
+            recommend_team = "requests"
+        # 最差情况：requests，dp有头和无头拿到的结果差距都很大，认定为有浏览器无头检测+动态网页
+        # if r_h_rate_diff < 40 and r_hless_rate_diff < 40 and h_hless_rate_diff < 40:
+        # 较差1：dp有头和无头差距很小，但是requests拿不到正确结果，认定为有requests防护 or 动态网页
+        elif h_hless_rate_diff < 30 and r_hless_rate_diff > 40:
+            recommend_team = "drissionpage_headless"
+        # 较差2：有头和无头差距很大，但是requests和有头拿到的结果基本一致，认定为要么有别的没有防护requests的waf，或者间歇性的瑞数【此时应该拿有头的cookie去判断其中是否有瑞数特征，上面已经做了】
+        # if r_h_rate_diff < 15 and h_hless_rate_diff > 40:
+        return dp_mcp_message_pack(
+            f"已完成tab页:【{tab_id}】的分析，该tab页存在waf",
+            tab_id=tab_id,
+            recommend_team=recommend_team,
+            raw_head_rate_difference=r_h_rate_diff,
+            raw_headless_rate_difference=h_hless_rate_diff,
+            head_headless_rate_difference=h_hless_rate_diff
+        )
+
+
+def register_click_action(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="click_action", description="尝试点击tab页中的元素，返回元素是否可以被点击，以及是否点击成功。")
+    async def click_action(browser_port: int, tab_id: str, css_selector: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        target_tab = _browser.get_tab(tab_id)
+        css_selector = css_selector
+        if "css:" not in css_selector:
+            css_selector = "css:" + css_selector
+        target_eles = target_tab.eles(css_selector)
+        click_success = False
+        element_clickable = False
+        if len(target_eles) == 1:
+            target_element = target_eles[0]
+            element_clickable = target_element.states.is_clickable
+            try:
+                target_element.click()
+                click_success = True
+            except Exception as e:
+                click_success = False
+            message = f"tab页:【{tab_id}】点击【{css_selector}】 {'成功' if click_success else '失败'} 了"
+        else:
+            message = f"tab页:【{tab_id}】传入的css_selector找到了{len(target_eles)}个元素，请确保传入的css_selector可以找到唯一的一个元素"
+        return dp_mcp_message_pack(
+            message=message,
+            browser_port=browser_port,
+            tab_id=tab_id,
+            css_selector=css_selector,
+            element_clickable=element_clickable,
+            click_success=click_success,
+            extra_message="点击成功，页面可能有更新，请重新获取页面html，并重新分析页面Selector" if click_success else ""
+        )
+
+
+def register_scroll_action(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="scroll_action", description="尝试滚动tab页"
+                                                "forward参数是滚动的方向：down、up、left、right"
+                                                "pixel参数是滚动的像素值，默认为None。"
+                                                "当forward为down且pixel为None，则将页面滚动到垂直中间位置，水平位置不变"
+                                                "当forward为up且pixel为None，则将页面滚动到顶部，水平位置不变"
+                                                "当forward为left且pixel为None，则将页面滚动到最左边，垂直位置不变"
+                                                "当forward为right且pixel为None，则将页面滚动到最右边，垂直位置不变")
+    async def scroll_action(browser_port: int, tab_id: str, forward: str = "down", pixel: int = None) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        target_tab = _browser.get_tab(tab_id)
+        if forward == "down":
+            if pixel is None:
+                target_tab.scroll.to_half()
+            target_tab.scroll.down(pixel)
+        elif forward == "up":
+            if pixel is None:
+                target_tab.scroll.to_top()
+            target_tab.scroll.up(pixel)
+        elif forward == "left":
+            if pixel is None:
+                target_tab.scroll.to_leftmost()
+            target_tab.scroll.left(pixel)
+        elif forward == "right":
+            if pixel is None:
+                target_tab.scroll.to_rightmost()
+            target_tab.scroll.right(pixel)
+        else:
+            if pixel is None:
+                target_tab.scroll.to_half()
+            target_tab.scroll.down()
+        message = f"已完成对tab页:【{tab_id}】forward={forward} 的滑动"
+        return dp_mcp_message_pack(
+            message=message,
+            browser_port=browser_port,
+            tab_id=tab_id,
+        )

{jarvis_brain-0.1.5.5 → jarvis_brain-0.1.10.0}/mcp_tools/main.py

@@ -2,6 +2,7 @@ from fastmcp import FastMCP
 
 from mcp_tools.dp_tools import *
 from tools.browser_manager import browser_manager
+from tools.browser_proxy import client_manager
 
 mcp = FastMCP("Jarvis Brain Mcp Tools")
 
@@ -13,18 +14,22 @@ if "TeamNode-Dp" in enabled_modules:
     # 页面管理
     register_close_tab(mcp, browser_manager)
     register_switch_tab(mcp, browser_manager)
-    register_get_new_tab(mcp, browser_manager)
-    # 功能
-    register_visit_url(mcp, browser_manager)
+    register_get_new_tab(mcp, browser_manager, client_manager)
+    # 基础功能
+    register_visit_url(mcp, browser_manager, client_manager)
     register_get_html(mcp, browser_manager)
     register_check_selector(mcp, browser_manager)
+    register_pop_first_packet(mcp, browser_manager, client_manager)
+    # 页面交互
+    register_click_action(mcp, browser_manager)
+    register_scroll_action(mcp, browser_manager)
 
 if "JarvisNode" in enabled_modules:
     register_assert_waf(mcp, browser_manager)
 
 
 def main():
-    mcp.run(transport="stdio")
+    mcp.run(transport="stdio",show_banner=False)
 
 
 if __name__ == '__main__':

{jarvis_brain-0.1.5.5 → jarvis_brain-0.1.10.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "Jarvis_Brain"  # 别人下载时用的名字，必须在 PyPI 上唯一
-version = "0.1.5.5"
+version = "0.1.10.0"
 description = "Jarvis brain mcp"
 dependencies = [
     "fastmcp",

{jarvis_brain-0.1.5.5 → jarvis_brain-0.1.10.0}/tools/browser_manager.py

@@ -3,6 +3,10 @@ import random
 from typing import Optional, Tuple
 import os
 from DrissionPage import ChromiumPage, ChromiumOptions
+import platform
+from DrissionPage.common import Settings
+
+Settings.set_raise_when_click_failed(True)
 
 
 class BrowserManager:
@@ -15,13 +19,17 @@ class BrowserManager:
             cls._instance.browser_pool = {}
         return cls._instance
 
-    def create_browser(self) -> Tuple[int, ChromiumPage]:
+    def create_browser(self, user_agent: str = None) -> Tuple[int, ChromiumPage]:
         """创建新的浏览器实例"""
         random_port = random.randint(9223, 9934)
         while random_port in self.browser_pool:
             random_port = random.randint(9223, 9934)
 
         co = ChromiumOptions().set_local_port(random_port)
+        if user_agent:
+            co.set_user_agent(user_agent)
+        if platform.system() != 'Windows':
+            co.set_argument('--no-sandbox')
         custom_data_dir = os.path.join(os.path.expanduser('~'), 'DrissionPage', "userData", f"{random_port}")
         co.set_user_data_path(custom_data_dir)  # 设置用户数据路径
         # if not os.path.exists(custom_data_dir):

jarvis_brain-0.1.10.0/tools/browser_proxy.py

@@ -0,0 +1,184 @@
+import threading
+from collections import deque
+import time
+from DrissionPage import ChromiumPage, ChromiumOptions
+from DrissionPage._pages.chromium_tab import ChromiumTab
+from DrissionPage._units.listener import DataPacket
+from typing import Tuple, Optional
+import json
+from urllib.parse import urlparse, urlunparse
+
+one_turn_max_token = 16000
+
+
+class DPProxyClient:
+    def __init__(self, driver: ChromiumTab, packet_filter: dict, self_kill=False):
+        self.tab_id = driver.tab_id
+        self.driver = ChromePageProxy(driver, self)
+        self.thread = None
+        self.self_kill = self_kill
+        self.packet_filter = packet_filter
+        self.packet_queue = deque()
+
+    def get_driver(self, start_listen, count=None, timeout=10) -> ChromiumTab:
+        """
+        获取代理后的driver、tab
+        :param start_listen: 若你自己写的代码里已经使用自动化框架监听发包的功能了，则该值应该置为False。若没监听，则必须将该值置为True
+        :param count: 需捕获的数据包总数，为None表示无限
+        :param timeout: 两个数据包之间等待的最大时长（秒），为None表示无限，默认为10秒
+        :return:
+        """
+        if start_listen:
+            self.driver.listen.set_targets(res_type=('xhr', 'fetch'))
+            self.driver.listen.start()
+            self.thread = threading.Thread(target=self.start_listen, args=(count, timeout,))
+            self.thread.start()
+        return self.driver
+
+    def start_listen(self, count=None, timeout=10):
+        for _ in self.driver.listen.steps(count=count, timeout=timeout, gap=1):
+            pass
+
+    # 每次调用函数，都从队列的左端弹出一个数据包
+    def pop_first_packet(self):
+        if self.packet_queue:
+            result = self.packet_queue.popleft()
+            current_queue_size = len(self.packet_queue)
+            return current_queue_size, json.dumps(result, ensure_ascii=False, separators=(',', ':')).replace("\\", "")
+        else:
+            return 0, None
+
+
+class DPProxyClientManager:
+    """浏览器池管理器 - 使用单例模式"""
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance.tab_pool = {}
+        return cls._instance
+
+    def create_client(self, tab: ChromiumTab, packet_filter: dict, self_kill=False) -> Tuple[
+        str, DPProxyClient, ChromiumTab]:
+        """创建新的tab页面代理实例"""
+        client = DPProxyClient(tab, packet_filter, self_kill=self_kill)
+        tab = client.get_driver(True, timeout=60 * 10)
+        tab_id = tab.tab_id
+        self.tab_pool[tab_id] = {"client": client, "driver": tab}
+        return tab_id, client, tab
+
+    def get_client(self, tab_id: str) -> Optional[DPProxyClient]:
+        """根据端口获取浏览器实例"""
+        return self.tab_pool.get(tab_id).get("client", None)
+
+    def remove_client(self, tab_id: str) -> Tuple[bool, Optional[ChromiumPage]]:
+        """根据端口移除浏览器实例"""
+        client = self.tab_pool.pop(tab_id, None)
+        return client is not None, client
+
+    def list_clients(self) -> list[int]:
+        """列出所有活跃的浏览器端口"""
+        return list(self.tab_pool.keys())
+
+
+class ChromePageProxy:
+    def __init__(self, page, client=None):
+        self.__dict__['page'] = page
+        self.__dict__['client'] = client
+
+    def __getattr__(self, item):
+        attr = getattr(self.page, item)
+        print(item, attr)
+        if item == 'listen':
+            listen_proxy = DrissionPageListenerProxy(attr, self.__dict__['client'])
+            return listen_proxy
+        return attr
+
+
+class DrissionPageListenerProxy:
+    def __init__(self, listener, client=None):
+        self.listener = listener
+        self.client = client
+
+    def __getattr__(self, item):
+        attr = getattr(self.listener, item)
+        # 当监听到wait被调用的时候
+        if item == "wait":
+            def wrapper(*args, **kwargs):
+                result = attr(*args, **kwargs)
+                check_data_packet(result, self.client)
+                return result
+
+            return wrapper
+        # 当监听到steps被调用的时候
+        if item == "steps":
+            def wrapper(*args, **kwargs):
+                if kwargs.get("gap", 1) > 1:
+                    raise Exception("暂不支持多包监控")
+                result = attr(*args, **kwargs)
+                if attr.__name__ == "steps":
+                    for step in result:
+                        check_data_packet(step, self.client)
+                        yield step
+
+            return wrapper
+        return attr
+
+
+def check_data_packet(packet: DataPacket, client: DPProxyClient):
+    """
+    封装监听到的数据包，并将其存放在client的packet_queue中
+    :param packet:
+    :param client:
+    :return:
+    """
+    url = packet.url
+    method = packet.request.method
+    data = None
+    if packet.request.hasPostData:
+        data = packet.request.postData
+    domain = urlparse(url).netloc
+    body = packet.response.body
+    body_str = json.dumps(body, ensure_ascii=False, separators=(',', ':'))
+    body_str_list = [body_str[i:i + one_turn_max_token] for i in range(0, len(body_str), one_turn_max_token)]
+    body_completed = True
+    packet_filter = client.packet_filter
+    domain_filter = packet_filter.get("domain_filter", None)
+    method_filter = packet_filter.get("method_filter", ["GET", "POST"])
+    for index, body_str in enumerate(body_str_list):
+        # 如果给了domain_filter并且domain没有在domain_filter中时跳过该数据包
+        if domain_filter and domain not in domain_filter:
+            continue
+        # 如果method没有在method_filter中，则跳过该数据包
+        if method not in method_filter:
+            continue
+        if (index + 1) != len(body_str_list):
+            body_completed = False
+        temp_dict = {
+            "url": url,
+            "body_completed": body_completed,
+            "method": method,
+            "request_data": data,
+            "request_headers": dict(packet.request.headers),
+            "response_headers": dict(packet.response.headers),
+            "response_body_segment": body_str.replace("\\", ""),
+        }
+        client.packet_queue.append(temp_dict)
+
+
+client_manager = DPProxyClientManager()
+
+# if __name__ == '__main__':
+#     co = ChromiumOptions().set_user_agent(
+#         "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36")
+#     tab = ChromiumPage(co).latest_tab
+#     client = DPProxyClient(tab, self_kill=False)
+#     # client = CaptchaClient(tab, self_kill=True)
+#     tab = client.get_driver(True)
+#     url = "https://api.toutiaoapi.com/feoffline/hotspot_and_local/html/hot_list/index.html?client_extra_params=%7B%22custom_log_pb%22%3A%22%7B%5C%22style_id%5C%22%3A%5C%2240030%5C%22%2C%5C%22entrance_hotspot%5C%22%3A%5C%22search%5C%22%2C%5C%22location%5C%22%3A%5C%22hot_board%5C%22%2C%5C%22category_name%5C%22%3A%5C%22hotboard_light%5C%22%7D%22%7D&count=50&log_pb=%7B%22style_id%22%3A%2240030%22%2C%22entrance_hotspot%22%3A%22search%22%2C%22location%22%3A%22hot_board%22%2C%22category_name%22%3A%22hotboard_light%22%7D&only_hot_list=1&tab_name=stream&enter_keyword=%23%E7%BE%8E%E5%9B%BD%E9%80%80%E5%87%BA66%E4%B8%AA%E5%9B%BD%E9%99%85%E7%BB%84%E7%BB%87%23"
+#     tab.get(url)
+#     for _ in range(5056):
+#         new_packet = client.pop_first_packet()
+#         print(new_packet, "23")
+#         time.sleep(1)

{jarvis_brain-0.1.5.5 → jarvis_brain-0.1.10.0}/tools/tools.py

@@ -7,6 +7,87 @@ from bs4 import BeautifulSoup
 from curl_cffi import requests
 from lxml import html, etree
 
+compress_html_js = """
+    function getSimplifiedDOM(node) {
+        // 1. 处理文本节点
+        if (node.nodeType === Node.TEXT_NODE) {
+            const text = node.textContent.trim();
+            // 限制文本长度，避免大段文章消耗 token，保留前100个字符通常足够定位
+            return text ? text.slice(0, 100) + (text.length > 100 ? '...' : '') : null;
+        }
+
+        // 2. 过滤无用标签
+        const ignoreTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'IFRAME', 'SVG', 'LINK', 'META'];
+        if (ignoreTags.includes(node.tagName)) return null;
+        if (node.nodeType !== Node.ELEMENT_NODE) return null;
+
+        // 3. 过滤不可见元素
+        const style = window.getComputedStyle(node);
+        if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') return null;
+        // 过滤宽高太小的元素（往往是埋点空像素）
+        const rect = node.getBoundingClientRect();
+        if (rect.width === 0 || rect.height === 0) return null;
+
+        // --- 开始构建标签字符串 ---
+        const tagName = node.tagName.toLowerCase();
+        let tagStr = tagName;
+
+        // A. 基础标识符 (ID 和 Class)
+        if (node.id) tagStr += `#${node.id}`;
+        if (node.className && typeof node.className === 'string') {
+            // 过滤掉 Tailwind 等太长且无语义的 class，保留有意义的业务 class
+            // 这里简单处理，全部保留，让 LLM 自己判断
+            const classes = node.className.trim().split(/\s+/);
+            if (classes.length > 0) tagStr += `.${classes.join('.')}`;
+        }
+
+        // B. 关键属性白名单 (这是你指出问题的核心修复)
+        const props = [];
+
+        // 通用重要属性
+        if (node.getAttribute('role')) props.push(`role="${node.getAttribute('role')}"`);
+        if (node.getAttribute('aria-label')) props.push(`aria-label="${node.getAttribute('aria-label')}"`);
+        if (node.getAttribute('title')) props.push(`title="${node.getAttribute('title')}"`);
+
+        // 特定标签的特定属性
+        if (tagName === 'a') {
+            const href = node.getAttribute('href');
+            // 只保留有意义的链接，忽略 javascript:;
+            if (href && !href.startsWith('javascript')) props.push(`href="${href}"`);
+        } else if (tagName === 'input' || tagName === 'textarea' || tagName === 'select') {
+            if (node.getAttribute('type')) props.push(`type="${node.getAttribute('type')}"`);
+            if (node.getAttribute('name')) props.push(`name="${node.getAttribute('name')}"`);
+            if (node.getAttribute('placeholder')) props.push(`placeholder="${node.getAttribute('placeholder')}"`);
+            if (node.disabled) props.push('disabled');
+            if (node.checked) props.push('checked');
+        } else if (tagName === 'button') {
+            if (node.getAttribute('type')) props.push(`type="${node.getAttribute('type')}"`);
+        } else if (tagName === 'img') {
+            if (node.getAttribute('alt')) props.push(`alt="${node.getAttribute('alt')}"`);
+        }
+
+        if (props.length > 0) {
+            tagStr += ` ${props.join(' ')}`;
+        }
+
+        // 4. 递归子节点
+        const children = Array.from(node.childNodes)
+            .map(getSimplifiedDOM)
+            .filter(n => n !== null);
+
+        // 5. 组装输出
+        // 如果没有子节点，也没有ID/Class，也不是输入框/图片/链接，那这个标签可能只是布局用的 div，可以考虑跳过它直接返回子节点内容
+        // 但为了保持结构完整，我们暂时保留它
+        if (children.length === 0) {
+            // 自闭合标签或空标签
+            return `<${tagStr} />`;
+        }
+        return `<${tagStr}>${children.join('')}</${tagName}>`; // 结束标签只保留 tagName 节省 token
+    }
+
+    return getSimplifiedDOM(document.body);
+"""
+
 
 # 使用requests获取html，用于测试是否使用了瑞数和jsl
 def requests_html(url):
@@ -53,6 +134,15 @@ def compress_html(content, only_text=False):
     for meta in doc.xpath('//meta'):
         meta.getparent().remove(meta)
 
+    for svg in doc.xpath('//svg'):
+        # 获取 SVG 内的文本内容
+        text_content = svg.text_content()
+        # 创建一个新的文本节点替换 SVG
+        parent = svg.getparent()
+        if parent is not None:
+            parent.text = (parent.text or '') + text_content
+            parent.remove(svg)
+
     # 删除 style 属性
     for element in doc.xpath('//*[@style]'):
         element.attrib.pop('style')

jarvis_brain-0.1.5.5/mcp_tools/dp_tools.py

@@ -1,166 +0,0 @@
-import json
-import os
-from typing import Any
-
-from DrissionPage._elements.none_element import NoneElement
-from fastmcp import FastMCP
-
-from tools.tools import compress_html, requests_html, dp_headless_html, assert_waf_cookie, dp_mcp_message_pack
-
-html_source_code_local_save_path = os.path.join(os.getcwd(), "html-source-code")
-waf_status_code_dict = {
-    412: "瑞数",
-    521: "加速乐"
-}
-
-
-def register_visit_url(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="visit_url", description="使用Drissionpage打开url访问某个网站")
-    async def visit_url(url: str) -> dict[str, Any]:
-        port, _browser = browser_manager.create_browser()
-        tab = _browser.get_tab()
-        tab.get(url)
-        tab_id = tab.tab_id
-        return dp_mcp_message_pack(
-            f"已在[{port}]端口创建浏览器对象，并已打开链接：{url}",
-            tab_id=tab_id,
-            browser_port=port
-        )
-
-
-def register_get_html(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="get_html", description="使用Drissionpage获取某一个tab页的html")
-    async def get_html(browser_port: int, tab_id: str) -> dict[str, Any]:
-        _browser = browser_manager.get_browser(browser_port)
-        tab = _browser.get_tab(tab_id)
-        file_name = tab.title + f"_{tab_id}.html"
-        if not os.path.exists(html_source_code_local_save_path):
-            os.makedirs(html_source_code_local_save_path)
-        abs_path = os.path.join(html_source_code_local_save_path, file_name)
-        with open(abs_path, "w", encoding="utf-8") as f:
-            min_html, compress_rate = compress_html(tab.html)
-            f.write(min_html)
-        return dp_mcp_message_pack(f"已保存tab页：【{tab_id}】的html源码", tab_id=tab_id, html_local_path=abs_path)
-
-
-def register_get_new_tab(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="get_new_tab", description="使用Drissionpage创建一个新的tab页，在新的tab页中打开url")
-    async def get_new_tab(browser_port: int, url: str) -> dict[str, Any]:
-        _browser = browser_manager.get_browser(browser_port)
-        tab = _browser.new_tab(url)
-        _browser.activate_tab(tab)
-        tab_id = tab.tab_id
-        return dp_mcp_message_pack(f"已创建新的tab页，并打开链接：{url}", tab_id=tab_id)
-
-
-def register_switch_tab(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="switch_tab", description="根据传入的tab_id切换到对应的tab页", )
-    async def switch_tab(browser_port: int, tab_id: str) -> dict[str, Any]:
-        _browser = browser_manager.get_browser(browser_port)
-        _browser.activate_tab(tab_id)
-        return dp_mcp_message_pack(f"已将tab页:【{tab_id}】切换至最前端")
-
-
-def register_close_tab(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="close_tab", description="根据传入的tab_id关闭tab页", )
-    async def close_tab(browser_port, tab_id) -> dict[str, Any]:
-        _browser = browser_manager.get_browser(browser_port)
-        _browser.close_tabs(tab_id)
-        return dp_mcp_message_pack(f"已将tab页:【{tab_id}】关闭")
-
-
-def register_check_selector(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="check_selector", description="查找tab页中是否包含元素，并返回元素attr_name所对应的值")
-    async def check_selector(browser_port: int, tab_id: str, css_selector: str, attr_name: str = "text") -> dict[str, Any]:
-        _browser = browser_manager.get_browser(browser_port)
-        target_tab = _browser.get_tab(tab_id)
-        css_selector = css_selector
-        if "css:" not in css_selector:
-            css_selector = "css:" + css_selector
-        target_eles = target_tab.eles(css_selector)
-        exist_flag = False
-        if len(target_eles) != 0:
-            exist_flag = True
-        if attr_name == "text":
-            ele_text_list = [i.text.replace("\n", "") for i in target_eles]
-            attr_output = "\n".join(ele_text_list)
-        else:
-            attr_output = json.dumps([i.attr(attr_name) for i in target_eles])
-        return dp_mcp_message_pack(
-            f"已完成tab页:【{tab_id}】对：【{css_selector}】的检查",
-            tab_id=tab_id,
-            selector=css_selector,
-            selector_ele_exist=exist_flag,
-            attr_output=attr_output
-        )
-
-
-def register_quit_browser(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="quit_browser", description="退出浏览器会话，关闭浏览器")
-    async def quit_browser(browser_port: int) -> dict[str, Any]:
-        flag, _browser = browser_manager.remove_page(browser_port)
-        if flag:
-            _browser.quit()
-        return dp_mcp_message_pack(
-            f"浏览器[{browser_port}]，退出会话，关闭浏览器{'成功' if flag else '失败'}",
-            browser_port=browser_port,
-            quit_flag=flag
-        )
-
-
-def register_assert_waf(mcp: FastMCP, browser_manager):
-    @mcp.tool(name="assert_waf",
-              description="通过对比requests、有头浏览器、无头浏览器获取到的html，判断网页是否使用了waf以及是否为动态渲染的网页")
-    async def assert_waf(browser_port: int, tab_id: str) -> dict[str, Any]:
-        _browser = browser_manager.get_browser(browser_port)
-        target_tab = _browser.get_tab(tab_id)
-        recommend_team = "drissionpage_head"
-        head_cookies = target_tab.cookies()
-        # 通过cookie判断是否有waf
-        waf_flag, waf_type = assert_waf_cookie(head_cookies)
-        head_html = target_tab.html
-        min_head_html, head_rate = compress_html(head_html, only_text=True)
-        raw_html, status_code = requests_html(target_tab.url)
-        min_raw_html, raw_rate = compress_html(raw_html, only_text=True)
-        r_h_rate_diff = abs(head_rate - raw_rate)
-        # 如果有已知的防火墙，则不浪费时间使用无头获取html和压缩比了
-        if waf_flag or status_code in waf_status_code_dict.keys():
-            return dp_mcp_message_pack(
-                f"已完成tab页:【{tab_id}】的分析，该tab页存在waf",
-                tab_id=tab_id,
-                recommend_team=recommend_team,
-                raw_head_rate_difference=r_h_rate_diff,
-                raw_headless_rate_difference=0,
-                head_headless_rate_difference=0
-            )
-
-        headless_html = dp_headless_html(target_tab.url)
-        min_headless_html, headless_rate = compress_html(headless_html, only_text=True)
-        r_hless_rate_diff = abs(raw_rate - headless_rate)
-        h_hless_rate_diff = abs(head_rate - headless_rate)
-        # 最优情况：requests，dp有头和无头拿到的结果基本一致，认定为没有防护的静态网页
-        if r_h_rate_diff < 40 and r_hless_rate_diff < 40 and h_hless_rate_diff < 40:
-            recommend_team = "requests"
-        # 最差情况：requests，dp有头和无头拿到的结果差距都很大，认定为有浏览器无头检测+动态网页
-        # if r_h_rate_diff < 40 and r_hless_rate_diff < 40 and h_hless_rate_diff < 40:
-        # 较差1：dp有头和无头差距很小，但是requests拿不到正确结果，认定为有requests防护 or 动态网页
-        elif h_hless_rate_diff < 30 and r_hless_rate_diff > 40:
-            recommend_team = "drissionpage_headless"
-        # 较差2：有头和无头差距很大，但是requests和有头拿到的结果基本一致，认定为要么有别的没有防护requests的waf，或者间歇性的瑞数【此时应该拿有头的cookie去判断其中是否有瑞数特征，上面已经做了】
-        # if r_h_rate_diff < 15 and h_hless_rate_diff > 40:
-        return dp_mcp_message_pack(
-            f"已完成tab页:【{tab_id}】的分析，该tab页存在waf",
-            tab_id=tab_id,
-            recommend_team=recommend_team,
-            raw_head_rate_difference=r_h_rate_diff,
-            raw_headless_rate_difference=h_hless_rate_diff,
-            head_headless_rate_difference=h_hless_rate_diff
-        )
-
-# def register_highlight_element_captcha(mcp: FastMCP, browser_manager):
-#     @mcp.tool(name="highlight_element_captcha",
-#               description="将传入的Selector在页面上高亮，并截屏")
-#     async def highlight_element_captcha(browser_port: int, tab_id: str, selector: str) -> dict[str, Any]:
-#         _browser = browser_manager.get_browser(browser_port)
-#         tab = _browser.get_tab(tab_id)
-#         tab.ele