Jarvis-Brain 0.1.6.1__py3-none-any.whl → 0.1.7.2__py3-none-any.whl

{jarvis_brain-0.1.6.1.dist-info → jarvis_brain-0.1.7.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Jarvis_Brain
-Version: 0.1.6.1
+Version: 0.1.7.2
 Summary: Jarvis brain mcp
 Requires-Python: >=3.10
 Requires-Dist: beautifulsoup4

jarvis_brain-0.1.7.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+mcp_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mcp_tools/dp_tools.py,sha256=LT2x9edWmmDqNoy66zPgwjUSLpYbMPUPluvDoAXs0FU,10421
+mcp_tools/main.py,sha256=Fdt2N3oKGwvruuno_ywnuWSlm1BexE9ZY669H2LTo9w,1056
+tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tools/browser_manager.py,sha256=EXM7n-sDOtdQGpWkVTAZHWhepVU-7PAoUTDNgGF9_fQ,1938
+tools/browser_proxy.py,sha256=yVTRpZA0IGLsL2Bl6zN_4hMdx2lX9UaMx5O6DPlrGRg,6333
+tools/tools.py,sha256=TaWs-CNXy-py9BFmCnJrQ09ke938xXpImf-N2Qo_Rvc,4708
+jarvis_brain-0.1.7.2.dist-info/METADATA,sha256=_j2fYANuNrAtnMwBnttjNSZ9z2vq7CDbjHS5LNnzKmQ,241
+jarvis_brain-0.1.7.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+jarvis_brain-0.1.7.2.dist-info/entry_points.txt,sha256=YFQT4xpkUqt5dM5wlKPQQOqcjMuFrT9iuRAzIpAyH7U,51
+jarvis_brain-0.1.7.2.dist-info/RECORD,,

mcp_tools/dp_tools.py

@@ -7,6 +7,7 @@ from fastmcp import FastMCP
 
 from tools.browser_manager import BrowserManager
 from tools.tools import compress_html, requests_html, dp_headless_html, assert_waf_cookie, dp_mcp_message_pack
+from tools.browser_proxy import DPProxyClient, DPProxyClientManager
 
 html_source_code_local_save_path = os.path.join(os.getcwd(), "html-source-code")
 waf_status_code_dict = {
@@ -17,7 +18,7 @@ waf_status_code_dict = {
 ont_turn_max_token = 25000
 
 
-def register_visit_url(mcp: FastMCP, browser_manager: BrowserManager):
+def register_visit_url(mcp: FastMCP, browser_manager: BrowserManager, client_manager: DPProxyClientManager):
     @mcp.tool(name="visit_url",
               description="使用Drissionpage打开url访问某个网站，并开始监听初始tab页的所有的XHR请求，当需要使用手机版浏览器Ua时use_mobile_user_agent为True")
     async def visit_url(url: str, use_mobile_user_agent=False) -> dict[str, Any]:
@@ -26,6 +27,7 @@ def register_visit_url(mcp: FastMCP, browser_manager: BrowserManager):
             mobile_user_agent = "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36"
         port, _browser = browser_manager.create_browser(mobile_user_agent)
         tab = _browser.get_tab()
+        client_manager.create_client(tab)
         tab.get(url)
         tab_id = tab.tab_id
         return dp_mcp_message_pack(
@@ -35,12 +37,12 @@ def register_visit_url(mcp: FastMCP, browser_manager: BrowserManager):
         )
 
 
-def register_get_html(mcp: FastMCP, browser_manager):
+def register_get_html(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
     @mcp.tool(name="get_html", description="使用Drissionpage获取某一个tab页的html")
     async def get_html(browser_port: int, tab_id: str) -> dict[str, Any]:
         _browser = browser_manager.get_browser(browser_port)
         tab = _browser.get_tab(tab_id)
-        file_name_prefix = str(tab.title).replace("/", "_").replace(":", "_")
+        file_name_prefix = hashlib.md5(str(tab.title).encode('utf-8')).hexdigest()
         if not os.path.exists(html_source_code_local_save_path):
             os.makedirs(html_source_code_local_save_path)
         min_html, compress_rate = compress_html(tab.html)
@@ -56,18 +58,20 @@ def register_get_html(mcp: FastMCP, browser_manager):
         return dp_mcp_message_pack(message, tab_id=tab_id, htmls_local_path=html_file_list)
 
 
-def register_get_new_tab(mcp: FastMCP, browser_manager):
+def register_get_new_tab(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
     @mcp.tool(name="get_new_tab",
               description="使用Drissionpage创建一个新的tab页，在新的tab页中打开url，并开始监听新的tab页的所有XHR请求")
     async def get_new_tab(browser_port: int, url: str) -> dict[str, Any]:
         _browser = browser_manager.get_browser(browser_port)
-        tab = _browser.new_tab(url)
+        tab = _browser.new_tab()
+        client_manager.create_client(tab)
+        tab.get(url)
         _browser.activate_tab(tab)
         tab_id = tab.tab_id
         return dp_mcp_message_pack(f"已创建新的tab页，并打开链接：{url}", tab_id=tab_id)
 
 
-def register_switch_tab(mcp: FastMCP, browser_manager):
+def register_switch_tab(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
     @mcp.tool(name="switch_tab", description="根据传入的tab_id切换到对应的tab页", )
     async def switch_tab(browser_port: int, tab_id: str) -> dict[str, Any]:
         _browser = browser_manager.get_browser(browser_port)
@@ -75,7 +79,7 @@ def register_switch_tab(mcp: FastMCP, browser_manager):
         return dp_mcp_message_pack(f"已将tab页:【{tab_id}】切换至最前端")
 
 
-def register_close_tab(mcp: FastMCP, browser_manager):
+def register_close_tab(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
     @mcp.tool(name="close_tab", description="根据传入的tab_id关闭tab页", )
     async def close_tab(browser_port, tab_id) -> dict[str, Any]:
         _browser = browser_manager.get_browser(browser_port)
@@ -83,7 +87,7 @@ def register_close_tab(mcp: FastMCP, browser_manager):
         return dp_mcp_message_pack(f"已将tab页:【{tab_id}】关闭")
 
 
-def register_check_selector(mcp: FastMCP, browser_manager):
+def register_check_selector(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
     @mcp.tool(name="check_selector", description="查找tab页中是否包含元素，并返回元素attr_name所对应的值")
     async def check_selector(browser_port: int, tab_id: str, css_selector: str, attr_name: str = "text") -> dict[
         str, Any]:
@@ -110,7 +114,7 @@ def register_check_selector(mcp: FastMCP, browser_manager):
         )
 
 
-def register_quit_browser(mcp: FastMCP, browser_manager):
+def register_quit_browser(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
     @mcp.tool(name="quit_browser", description="退出浏览器会话，关闭浏览器")
     async def quit_browser(browser_port: int) -> dict[str, Any]:
         flag, _browser = browser_manager.remove_page(browser_port)
@@ -123,7 +127,25 @@ def register_quit_browser(mcp: FastMCP, browser_manager):
         )
 
 
-def register_assert_waf(mcp: FastMCP, browser_manager):
+def register_pop_first_packet(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
+    @mcp.tool(name="pop_first_packet",
+              description="每调用一次就会弹出传入的tab页所监听到的数据包中的第一个packet_message")
+    async def pop_first_packet(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
+        client = client_manager.get_client(tab_id)
+        packet_message = client.pop_first_packet()
+        message = f"tab页:【{tab_id}】，暂时没有监听到XHR数据包"
+        if packet_message:
+            message = f"tab页:【{tab_id}】，监听到XHR数据包",
+        return dp_mcp_message_pack(
+            message,
+            browser_port=browser_port,
+            tab_id=tab_id,
+            packet_message=packet_message
+        )
+
+
+def register_assert_waf(mcp: FastMCP, browser_manager, client_manager: DPProxyClientManager):
     @mcp.tool(name="assert_waf",
               description="通过对比requests、有头浏览器、无头浏览器获取到的html，判断网页是否使用了waf以及是否为动态渲染的网页")
     async def assert_waf(browser_port: int, tab_id: str) -> dict[str, Any]:

mcp_tools/main.py

@@ -2,6 +2,7 @@ from fastmcp import FastMCP
 
 from mcp_tools.dp_tools import *
 from tools.browser_manager import browser_manager
+from tools.browser_proxy import client_manager
 
 mcp = FastMCP("Jarvis Brain Mcp Tools")
 
@@ -11,16 +12,17 @@ base_cwd = os.getenv("BASE_CWD", os.path.expanduser('~'))
 
 if "TeamNode-Dp" in enabled_modules:
     # 页面管理
-    register_close_tab(mcp, browser_manager)
-    register_switch_tab(mcp, browser_manager)
-    register_get_new_tab(mcp, browser_manager)
+    register_close_tab(mcp, browser_manager, client_manager)
+    register_switch_tab(mcp, browser_manager, client_manager)
+    register_get_new_tab(mcp, browser_manager, client_manager)
     # 功能
-    register_visit_url(mcp, browser_manager)
-    register_get_html(mcp, browser_manager)
-    register_check_selector(mcp, browser_manager)
+    register_visit_url(mcp, browser_manager, client_manager)
+    register_get_html(mcp, browser_manager, client_manager)
+    register_check_selector(mcp, browser_manager, client_manager)
+    register_pop_first_packet(mcp, browser_manager, client_manager)
 
 if "JarvisNode" in enabled_modules:
-    register_assert_waf(mcp, browser_manager)
+    register_assert_waf(mcp, browser_manager, client_manager)
 
 
 def main():

tools/browser_proxy.py

@@ -0,0 +1,163 @@
+import threading
+from collections import deque
+import time
+from DrissionPage import ChromiumPage, ChromiumOptions
+from DrissionPage._pages.chromium_tab import ChromiumTab
+from DrissionPage._units.listener import DataPacket
+from typing import Tuple, Optional
+import json
+
+
+class DPProxyClient:
+    def __init__(self, driver: ChromiumTab, self_kill=False):
+        self.tab_id = driver.tab_id
+        self.driver = ChromePageProxy(driver, self)
+        self.thread = None
+        self.self_kill = self_kill
+        # self.packet_list = []
+        self.packet_queue = deque()
+
+    def get_driver(self, start_listen, count=None, timeout=10) -> ChromiumTab:
+        """
+        获取代理后的driver、tab
+        :param start_listen: 若你自己写的代码里已经使用自动化框架监听发包的功能了，则该值应该置为False。若没监听，则必须将该值置为True
+        :param count: 需捕获的数据包总数，为None表示无限
+        :param timeout: 两个数据包之间等待的最大时长（秒），为None表示无限，默认为10秒
+        :return:
+        """
+        if start_listen:
+            self.driver.listen.set_targets(res_type="XHR")
+            self.driver.listen.start()
+            self.thread = threading.Thread(target=self.start_listen, args=(count, timeout,))
+            self.thread.start()
+        return self.driver
+
+    def start_listen(self, count=None, timeout=10):
+        for _ in self.driver.listen.steps(count=count, timeout=timeout, gap=1):
+            pass
+
+    # 每次调用函数，都从队列的左端弹出一个数据包
+    def pop_first_packet(self):
+        if self.packet_queue:
+            result = self.packet_queue.popleft()
+            return json.dumps(result, ensure_ascii=False)
+        else:
+            return None
+
+
+class DPProxyClientManager:
+    """浏览器池管理器 - 使用单例模式"""
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance.tab_pool = {}
+        return cls._instance
+
+    def create_client(self, tab: ChromiumTab, self_kill=False) -> Tuple[str, DPProxyClient, ChromiumTab]:
+        """创建新的tab页面代理实例"""
+        client = DPProxyClient(tab, self_kill=self_kill)
+        tab = client.get_driver(True)
+        tab_id = tab.tab_id
+        self.tab_pool[tab_id] = {"client": client, "driver": tab}
+        return tab_id, client, tab
+
+    def get_client(self, tab_id: str) -> Optional[DPProxyClient]:
+        """根据端口获取浏览器实例"""
+        return self.tab_pool.get(tab_id).get("client", None)
+
+    def remove_client(self, tab_id: str) -> Tuple[bool, Optional[ChromiumPage]]:
+        """根据端口移除浏览器实例"""
+        client = self.tab_pool.pop(tab_id, None)
+        return client is not None, client
+
+    def list_clients(self) -> list[int]:
+        """列出所有活跃的浏览器端口"""
+        return list(self.tab_pool.keys())
+
+
+class ChromePageProxy:
+    def __init__(self, page, client=None):
+        self.__dict__['page'] = page
+        self.__dict__['client'] = client
+
+    def __getattr__(self, item):
+        attr = getattr(self.page, item)
+        print(item, attr)
+        if item == 'listen':
+            listen_proxy = DrissionPageListenerProxy(attr, self.__dict__['client'])
+            return listen_proxy
+        return attr
+
+
+class DrissionPageListenerProxy:
+    def __init__(self, listener, client=None):
+        self.listener = listener
+        self.client = client
+
+    def __getattr__(self, item):
+        attr = getattr(self.listener, item)
+        # 当监听到wait被调用的时候
+        if item == "wait":
+            def wrapper(*args, **kwargs):
+                result = attr(*args, **kwargs)
+                check_data_packet(result, self.client)
+                return result
+
+            return wrapper
+        # 当监听到steps被调用的时候
+        if item == "steps":
+            def wrapper(*args, **kwargs):
+                if kwargs.get("gap", 1) > 1:
+                    raise Exception("暂不支持多包监控")
+                result = attr(*args, **kwargs)
+                if attr.__name__ == "steps":
+                    for step in result:
+                        check_data_packet(step, self.client)
+                        yield step
+
+            return wrapper
+        return attr
+
+
+def check_data_packet(packet: DataPacket, client: DPProxyClient):
+    """
+    封装监听到的数据包，并将其存放在client的packet_queue中
+    :param packet:
+    :param client:
+    :return:
+    """
+    url = packet.url
+    method = packet.request.method
+    data = None
+    if packet.request.hasPostData:
+        data = packet.request.postData
+    body = packet.response.body
+    temp_dict = {
+        "url": url,
+        "method": method,
+        "request_data": data,
+        "request_headers": dict(packet.request.headers),
+        "response_headers": dict(packet.response.headers),
+        "response_body": body,
+    }
+    # print(temp_dict)
+    client.packet_queue.append(temp_dict)
+
+
+client_manager = DPProxyClientManager()
+
+if __name__ == '__main__':
+    co = ChromiumOptions().set_user_agent(
+        "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36")
+    tab = ChromiumPage(co).latest_tab
+    client = DPProxyClient(tab, self_kill=False)
+    # client = CaptchaClient(tab, self_kill=True)
+    tab = client.get_driver(True)
+    url = "https://api.toutiaoapi.com/feoffline/hotspot_and_local/html/hot_list/index.html?client_extra_params=%7B%22custom_log_pb%22%3A%22%7B%5C%22style_id%5C%22%3A%5C%2240030%5C%22%2C%5C%22entrance_hotspot%5C%22%3A%5C%22search%5C%22%2C%5C%22location%5C%22%3A%5C%22hot_board%5C%22%2C%5C%22category_name%5C%22%3A%5C%22hotboard_light%5C%22%7D%22%7D&count=50&log_pb=%7B%22style_id%22%3A%2240030%22%2C%22entrance_hotspot%22%3A%22search%22%2C%22location%22%3A%22hot_board%22%2C%22category_name%22%3A%22hotboard_light%22%7D&only_hot_list=1&tab_name=stream&enter_keyword=%23%E7%BE%8E%E5%9B%BD%E9%80%80%E5%87%BA66%E4%B8%AA%E5%9B%BD%E9%99%85%E7%BB%84%E7%BB%87%23"
+    tab.get(url)
+    for _ in range(5056):
+        new_packet = client.pop_first_packet()
+        print(new_packet, "23")
+        time.sleep(1)

jarvis_brain-0.1.6.1.dist-info/RECORD

@@ -1,10 +0,0 @@
-mcp_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mcp_tools/dp_tools.py,sha256=hijWdWQ9fhAQXrtrtqkaUd3CUH5sORAoQaF7BVtljfY,9078
-mcp_tools/main.py,sha256=SiLF-tcEdApkzMjTZJEpVYX5d-LPW7aqy6A6lCBA7YQ,829
-tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tools/browser_manager.py,sha256=EXM7n-sDOtdQGpWkVTAZHWhepVU-7PAoUTDNgGF9_fQ,1938
-tools/tools.py,sha256=TaWs-CNXy-py9BFmCnJrQ09ke938xXpImf-N2Qo_Rvc,4708
-jarvis_brain-0.1.6.1.dist-info/METADATA,sha256=1CSoEWNLJzbP1L_4Gz3ElX2jANUxnsMh8EqtwM5D6Pw,241
-jarvis_brain-0.1.6.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-jarvis_brain-0.1.6.1.dist-info/entry_points.txt,sha256=YFQT4xpkUqt5dM5wlKPQQOqcjMuFrT9iuRAzIpAyH7U,51
-jarvis_brain-0.1.6.1.dist-info/RECORD,,