Jarvis-Brain 0.1.3__py3-none-any.whl → 0.1.4.1__py3-none-any.whl

{jarvis_brain-0.1.3.dist-info → jarvis_brain-0.1.4.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Jarvis_Brain
-Version: 0.1.3
+Version: 0.1.4.1
 Summary: Jarvis brain
 Requires-Python: >=3.10
 Requires-Dist: beautifulsoup4

jarvis_brain-0.1.4.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+mcp_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mcp_tools/dp_tools.py,sha256=PwLd1zTFIQ673-cZjdON4TfU854_h5BgLPOVRbt_kVw,8349
+mcp_tools/main.py,sha256=WO9kNpIORRrIOAWW8jiAd3gNW6rFMExln8y4CquKrM8,837
+tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tools/browser_manager.py,sha256=AM9hIQKtgMVxIZsEPxfHj9q41ZtzHrOS69wtgGBE3-Q,1458
+tools/tools.py,sha256=3vTMCT_h0eDPtOMHc2KnhNVpQBUs6jgIChutBZZFMK4,5008
+jarvis_brain-0.1.4.1.dist-info/METADATA,sha256=jJjlYvPAjlzahJ9OtI6EuL1JOoAk5_OUku2XtqXeztE,277
+jarvis_brain-0.1.4.1.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+jarvis_brain-0.1.4.1.dist-info/entry_points.txt,sha256=YFQT4xpkUqt5dM5wlKPQQOqcjMuFrT9iuRAzIpAyH7U,51
+jarvis_brain-0.1.4.1.dist-info/RECORD,,

mcp_tools/dp_tools.py

@@ -1,231 +1,169 @@
-import json
 import os
-import random
 from typing import Any
 
-from urllib.parse import urlparse
-from DrissionPage import ChromiumPage, ChromiumOptions
 from DrissionPage._elements.none_element import NoneElement
 from fastmcp import FastMCP
 
-from tools.tools import compress_html, requests_html
-from tools.simhash_tools import HTMLSimHashComparator
+from tools.tools import compress_html, requests_html, dp_headless_html, assert_waf_cookie, dp_mcp_message_pack
+import psutil
+from pathlib import Path
 
 html_source_code_local_save_path = os.path.join(os.getcwd(), "html-source-code")
-browser_pool = {}
+waf_status_code_dict = {
+    412: "瑞数",
+    521: "加速乐"
+}
 
 
-# 随机一个浏览器池中不存在的端口，创建一个浏览器，返回随机端口，和浏览器对象。
-def create_browser():
-    global browser_pool
-    random_port = random.randint(9222, 9934)
-    while random_port in browser_pool:
-        random_port = random.randint(9222, 9934)
-    co = ChromiumOptions().set_local_port(random_port)
-    browser_pool[random_port] = ChromiumPage(co)
-    return random_port, browser_pool[random_port]
-
-
-# 根据传入的端口查找对应的浏览器对象
-def get_page(port):
-    return browser_pool.get(port, None)
-
-
-# 根据传入的端口查找并弹出一个浏览器对象
-def remove_page(port):
-    browser = browser_pool.pop(port, None)
-    return browser is not None, browser
-
-
-def register_visit_url(mcp: FastMCP):
+def register_visit_url(mcp: FastMCP, browser_manager):
     @mcp.tool(name="visit_url", description="使用Drissionpage打开url访问某个网站")
     async def visit_url(url: str) -> dict[str, Any]:
-        port, _browser = create_browser()
+        port, _browser = browser_manager.create_browser()
         tab = _browser.get_tab()
         tab.get(url)
         tab_id = tab.tab_id
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-                    "message": f"已在[{port}]端口创建浏览器对象，并已打开链接：{url}",
-                    "tab_id": tab_id,
-                    "browser_port": port,
-                }, ensure_ascii=False)
-            }]
-        }
-
-
-def register_get_html(mcp: FastMCP):
+        return dp_mcp_message_pack(
+            f"已在[{port}]端口创建浏览器对象，并已打开链接：{url}",
+            tab_id=tab_id,
+            browser_port=port
+        )
+
+
+def register_get_html(mcp: FastMCP, browser_manager):
     @mcp.tool(name="get_html", description="使用Drissionpage获取某一个tab页的html")
     async def get_html(browser_port: int, tab_id: str) -> dict[str, Any]:
-        _browser = get_page(browser_port)
+        _browser = browser_manager.get_browser(browser_port)
         tab = _browser.get_tab(tab_id)
         file_name = tab.title + f"_{tab_id}.html"
         if not os.path.exists(html_source_code_local_save_path):
             os.makedirs(html_source_code_local_save_path)
         abs_path = os.path.join(html_source_code_local_save_path, file_name)
         with open(abs_path, "w", encoding="utf-8") as f:
-            f.write(compress_html(tab.html))
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-                    "message": f"已保存tab页：【{tab_id}】的html源码",
-                    "tab_id": tab_id,
-                    "html_local_path": abs_path
-                }, ensure_ascii=False)
-            }]
-        }
-
-
-def register_get_new_tab(mcp: FastMCP):
+            min_html, compress_rate = compress_html(tab.html)
+            f.write(min_html)
+        return dp_mcp_message_pack(f"已保存tab页：【{tab_id}】的html源码", tab_id=tab_id, html_local_path=abs_path)
+
+
+def register_get_new_tab(mcp: FastMCP, browser_manager):
     @mcp.tool(name="get_new_tab", description="使用Drissionpage创建一个新的tab页，在新的tab页中打开url")
     async def get_new_tab(browser_port: int, url: str) -> dict[str, Any]:
-        _browser = get_page(browser_port)
+        _browser = browser_manager.get_browser(browser_port)
         tab = _browser.new_tab(url)
         _browser.activate_tab(tab)
         tab_id = tab.tab_id
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-                    "message": f"已创建新的tab页，并打开链接：{url}",
-                    "tab_id": tab_id,
-                }, ensure_ascii=False)
-            }]
-        }
-
-
-def register_switch_tab(mcp: FastMCP):
+        return dp_mcp_message_pack(f"已创建新的tab页，并打开链接：{url}", tab_id=tab_id)
+
+
+def register_switch_tab(mcp: FastMCP, browser_manager):
     @mcp.tool(name="switch_tab", description="根据传入的tab_id切换到对应的tab页", )
     async def switch_tab(browser_port: int, tab_id: str) -> dict[str, Any]:
-        _browser = get_page(browser_port)
+        _browser = browser_manager.get_browser(browser_port)
         _browser.activate_tab(tab_id)
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-                    "message": f"已将tab页:【{tab_id}】切换至最前端",
-                }, ensure_ascii=False)
-            }]
-        }
+        return dp_mcp_message_pack(f"已将tab页:【{tab_id}】切换至最前端")
 
 
-def register_close_tab(mcp: FastMCP):
+def register_close_tab(mcp: FastMCP, browser_manager):
     @mcp.tool(name="close_tab", description="根据传入的tab_id关闭tab页", )
     async def close_tab(browser_port, tab_id) -> dict[str, Any]:
-        _browser = get_page(browser_port)
+        _browser = browser_manager.get_browser(browser_port)
         _browser.close_tabs(tab_id)
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-                    "message": f"已将tab页:【{tab_id}】关闭",
-                }, ensure_ascii=False)
-            }]
-        }
-
-
-def register_check_selector(mcp: FastMCP):
-    @mcp.tool(name="check_selector", description="查找tab页中是否包含元素")
+        return dp_mcp_message_pack(f"已将tab页:【{tab_id}】关闭")
+
+
+def register_check_selector(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="check_selector", description="查找tab页中是否包含某个元素")
     async def check_selector(browser_port: int, tab_id: str, css_selector: str) -> dict[str, Any]:
-        _browser = get_page(browser_port)
+        _browser = browser_manager.get_browser(browser_port)
         target_tab = _browser.get_tab(tab_id)
         css_selector = css_selector
         if "css:" not in css_selector:
             css_selector = "css:" + css_selector
         target_ele = target_tab.ele(css_selector)
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-                    "message": f"已完成tab页:【{tab_id}】对：【{css_selector}】的检查",
-                    "tab_id": tab_id,
-                    "selector": css_selector,
-                    "selector_ele_exist": not isinstance(target_ele, NoneElement),
-                }, ensure_ascii=False)
-            }]
-        }
-
-
-def register_quit_browser(mcp: FastMCP):
+        return dp_mcp_message_pack(
+            f"已完成tab页:【{tab_id}】对：【{css_selector}】的检查",
+            tab_id=tab_id,
+            selector=css_selector,
+            selector_ele_exist=not isinstance(target_ele, NoneElement)
+        )
+
+
+def register_quit_browser(mcp: FastMCP, browser_manager):
     @mcp.tool(name="quit_browser", description="退出浏览器会话，关闭浏览器")
     async def quit_browser(browser_port: int) -> dict[str, Any]:
-        flag, _browser = remove_page(browser_port)
+        flag, _browser = browser_manager.remove_page(browser_port)
         if flag:
             _browser.quit()
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-                    "message": f"浏览器[{browser_port}]，退出会话，关闭浏览器{'成功' if flag else '失败'}",
-                    "browser_port": browser_port,
-                    "quit_flag": flag,
-                }, ensure_ascii=False)
-            }]
-        }
-
-
-def register_assert_Static_Web(mcp: FastMCP):
-    @mcp.tool(name="assert_Static_Web", description="判断tab页中的网页是否是静态网页")
-    def assert_Static_Web(browser_port: int, tab_id: str) -> dict[str, Any]:
-        _browser = get_page(browser_port)
+        return dp_mcp_message_pack(
+            f"浏览器[{browser_port}]，退出会话，关闭浏览器{'成功' if flag else '失败'}",
+            browser_port=browser_port,
+            quit_flag=flag
+        )
+
+
+def register_assert_waf(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="assert_waf",
+              description="通过对比requests、有头浏览器、无头浏览器获取到的html，判断网页是否使用了waf以及是否为动态渲染的网页")
+    async def assert_waf(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
         target_tab = _browser.get_tab(tab_id)
-        target_url = target_tab.url
-        raw_html, stat_code = requests_html(target_url)
-        if stat_code != 200:
-            return {
-                "content": [{
-                    "type": "text",
-                    "text": json.dumps({
-                        "messages": f"已完成tab页:【{tab_id}】的分析，该tab页使用requests获取的html状态码为[{stat_code}]，并非200，请使用assert_waf判断是否有waf",
-                        "tab_id": tab_id,
-                        "url": target_url,
-                        "stat_code": stat_code,
-                    }, ensure_ascii=False)
-                }]
-            }
-        render_html = target_tab.html
-        comparator = HTMLSimHashComparator(raw_html, render_html, False)
-        result = comparator.compare_simhash()
-        print("SimHash比较结果:")
-        print(f"页面1 SimHash: {result['simhash1']}")
-        print(f"页面2 SimHash: {result['simhash2']}")
-        print(f"汉明距离: {result['hamming_distance']}")
-        print(f"相似度: {result['similarity_percentage']}")
-        threshold_result = comparator.compare_with_threshold(threshold=0.7)
-        print(f"\n基于阈值{threshold_result['threshold']}的判断:")
-        print(f"是否相似: {threshold_result['is_similar']}")
-        static_html_flag = threshold_result['is_similar']
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-                    "message": f"已完成tab页:【{tab_id}】的分析，该tab页静态页面和渲染页面的相似度为[{result['similarity_percentage']}]，判定为 {'静态页面' if static_html_flag else '动态渲染'}",
-                    "tab_id": tab_id,
-                    "is_static_web": static_html_flag,
-                    "static_web_possibility": result['similarity_percentage'],
-                }, ensure_ascii=False)
-            }]
-        }
-
-
-def register_get_cookies(mcp: FastMCP):
-    @mcp.tool(name="get_cookies")
-    def get_cookies(browser_port: int, tab_id: str) -> dict[str, Any]:
-        _browser = get_page(browser_port)
+        recommend_team = "drissionpage_head"
+        head_cookies = target_tab.cookies()
+        # 通过cookie判断是否有waf
+        waf_flag, waf_type = assert_waf_cookie(head_cookies)
+        head_html = target_tab.html
+        min_head_html, head_rate = compress_html(head_html, only_text=True)
+        raw_html, status_code = requests_html(target_tab.url)
+        min_raw_html, raw_rate = compress_html(raw_html, only_text=True)
+        r_h_rate_diff = abs(head_rate - raw_rate)
+        # 如果有已知的防火墙，则不浪费时间使用无头获取html和压缩比了
+        if waf_flag or status_code in waf_status_code_dict.keys():
+            return dp_mcp_message_pack(
+                f"已完成tab页:【{tab_id}】的分析，该tab页存在waf",
+                tab_id=tab_id,
+                recommend_team=recommend_team,
+                raw_head_rate_difference=r_h_rate_diff,
+                raw_headless_rate_difference=0,
+                head_headless_rate_difference=0
+            )
+
+        headless_html = dp_headless_html(target_tab.url)
+        min_headless_html, headless_rate = compress_html(headless_html, only_text=True)
+        r_hless_rate_diff = abs(raw_rate - headless_rate)
+        h_hless_rate_diff = abs(head_rate - headless_rate)
+        # 最优情况：requests，dp有头和无头拿到的结果基本一致，认定为没有防护的静态网页
+        if r_h_rate_diff < 40 and r_hless_rate_diff < 40 and h_hless_rate_diff < 40:
+            recommend_team = "requests"
+        # 最差情况：requests，dp有头和无头拿到的结果差距都很大，认定为有浏览器无头检测+动态网页
+        # if r_h_rate_diff < 40 and r_hless_rate_diff < 40 and h_hless_rate_diff < 40:
+        # 较差1：dp有头和无头差距很小，但是requests拿不到正确结果，认定为有requests防护 or 动态网页
+        elif h_hless_rate_diff < 30 and r_hless_rate_diff > 40:
+            recommend_team = "drissionpage_headless"
+        # 较差2：有头和无头差距很大，但是requests和有头拿到的结果基本一致，认定为要么有别的没有防护requests的waf，或者间歇性的瑞数【此时应该拿有头的cookie去判断其中是否有瑞数特征，上面已经做了】
+        # if r_h_rate_diff < 15 and h_hless_rate_diff > 40:
+        return dp_mcp_message_pack(
+            f"已完成tab页:【{tab_id}】的分析，该tab页存在waf",
+            tab_id=tab_id,
+            recommend_team=recommend_team,
+            raw_head_rate_difference=r_h_rate_diff,
+            raw_headless_rate_difference=h_hless_rate_diff,
+            head_headless_rate_difference=h_hless_rate_diff
+        )
+
+
+def register_highlight_element_captcha(mcp: FastMCP, browser_manager):
+    @mcp.tool(name="highlight_element_captcha",
+              description="将传入的Selector在页面上高亮，并截屏")
+    async def highlight_element_captcha(browser_port: int, tab_id: str, selector: str) -> dict[str, Any]:
+        _browser = browser_manager.get_browser(browser_port)
         tab = _browser.get_tab(tab_id)
-        target_url = tab.url
-        parser = urlparse(target_url)
-        domain = parser.netloc
-        cookies = tab.cookies()
-        # tab.cookies
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps({
-
-                }, ensure_ascii=False)
-            }]
-        }
+        tab.ele
+
+
+def register_test(mcp: FastMCP, cwd: str):
+    @mcp.tool(name="test", description="用户说要进行测试时调用这个函数")
+    async def test() -> dict[str, Any]:
+        test_project = "获取根目录"
+        return dp_mcp_message_pack(
+            f"当前测试项目是：{test_project}",
+            result=cwd
+        )

mcp_tools/main.py

@@ -1,24 +1,25 @@
 # main.py
 from mcp_tools.dp_tools import *
-from mcp_tools.requests_tools import *
 from fastmcp import FastMCP
+from tools.browser_manager import browser_manager
 
 mcp = FastMCP("Jarvis Brain Mcp Tools")
 
 # 根据环境变量加载模块
 enabled_modules = os.getenv("MCP_MODULES", "TeamNode-Dp").split(",")
+base_cwd = os.getenv("BASE_CWD", os.environ.get("PWD"))
 
 if "TeamNode-Dp" in enabled_modules:
-    register_visit_url(mcp)
-    register_close_tab(mcp)
-    register_switch_tab(mcp)
-    register_get_html(mcp)
-    register_get_new_tab(mcp)
-    register_check_selector(mcp)
+    register_visit_url(mcp, browser_manager)
+    register_close_tab(mcp, browser_manager)
+    register_switch_tab(mcp, browser_manager)
+    register_get_html(mcp, browser_manager)
+    register_get_new_tab(mcp, browser_manager)
+    register_check_selector(mcp, browser_manager)
 
 if "JarvisNode" in enabled_modules:
-    register_assert_waf(mcp)
-    register_assert_Static_Web(mcp)
+    register_assert_waf(mcp, browser_manager)
+    register_test(mcp, base_cwd)
 
 
 def main():

tools/browser_manager.py

@@ -0,0 +1,42 @@
+"""浏览器池管理模块 - 单例模式确保状态共享"""
+import random
+from DrissionPage import ChromiumPage, ChromiumOptions
+from typing import Optional, Tuple
+
+
+class BrowserManager:
+    """浏览器池管理器 - 使用单例模式"""
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            cls._instance = super().__new__(cls)
+            cls._instance.browser_pool = {}
+        return cls._instance
+
+    def create_browser(self) -> Tuple[int, ChromiumPage]:
+        """创建新的浏览器实例"""
+        random_port = random.randint(9223, 9934)
+        while random_port in self.browser_pool:
+            random_port = random.randint(9223, 9934)
+
+        co = ChromiumOptions().set_local_port(random_port)
+        self.browser_pool[random_port] = ChromiumPage(co)
+        return random_port, self.browser_pool[random_port]
+
+    def get_browser(self, port: int) -> Optional[ChromiumPage]:
+        """根据端口获取浏览器实例"""
+        return self.browser_pool.get(port)
+
+    def remove_browser(self, port: int) -> Tuple[bool, Optional[ChromiumPage]]:
+        """根据端口移除浏览器实例"""
+        browser = self.browser_pool.pop(port, None)
+        return browser is not None, browser
+
+    def list_browsers(self) -> list[int]:
+        """列出所有活跃的浏览器端口"""
+        return list(self.browser_pool.keys())
+
+
+# 创建全局单例实例
+browser_manager = BrowserManager()

tools/tools.py

@@ -1,4 +1,5 @@
 import time
+import json
 
 import htmlmin
 from curl_cffi import requests
@@ -35,42 +36,15 @@ def requests_html(url):
 def dp_headless_html(url):
     opt = ChromiumOptions().headless(True)
     opt.set_argument('--no-sandbox')
-    opt.set_local_port(9976)
-
+    # 随机端口，固定9222端口的话，可能会被瑞数检测到
+    opt.auto_port()
     page = ChromiumPage(opt)
-    # page = ChromiumPage()
-
     tab = page.latest_tab
-    # tab.set.load_mode.normal()
     tab.get(url)
-    # tab.wait.eles_loaded()
+    # todo: 目前没有更好的方式，为了数据渲染完全，只能硬等【受网速波动影响比较大】
     time.sleep(10)
-    # tab.wait.load_start()
     page_html = tab.html
-    # print(page_html)
-    # print("dp_cookies=>", tab.cookies())
-    page.quit()
-    return page_html
-
-
-# 使用dp无头模式获取html，用于测试是否使用了其他waf，如移动waf
-def dp_html(url):
-    opt = ChromiumOptions()
-    opt.set_local_port(9223)
-    # opt.set_argument('--no-sandbox')
-    page = ChromiumPage(opt)
-    # page = ChromiumPage()
-
-    tab = page.latest_tab
-    # tab.set.load_mode.normal()
-    tab.get(url)
-    # tab.wait.eles_loaded()
-    # time.sleep(10)
-
-    tab.wait.doc_loaded()
-    page_html = tab.html
-    # print(page_html)
-    # print("dp_cookies=>", tab.cookies())
+    # 无头浏览器在用完之后一定要记得再page级别进行quit
     page.quit()
     return page_html
 
@@ -102,62 +76,44 @@ def compress_html(content, only_text=False):
 
     result = etree.tostring(doc, encoding='unicode')
     result = htmlmin.minify(result)
-    print(f"html压缩比=> {len(content) / len(result) * 100:.2f}%")
+    compress_rate = round(len(content) / len(result) * 100)
+    print(f"html压缩比=> {compress_rate}%")
     if not only_text:
-        return result
+        return result, compress_rate
     soup = BeautifulSoup(result, 'html.parser')
     result = soup.get_text(strip=True)
-    return result
-
-
-def test(target_url):
-    raw_html, status_code = requests_html(target_url)
-    raw_html = compress_html(raw_html, only_text=True)
-    # print("raw_html=>", status_code, compress_html(raw_html))
-    print("raw_html=>", status_code, raw_html)
-    page_html = dp_html(target_url)
-    page_html = compress_html(page_html, only_text=True)
-    # print("render_text=>", compress_html(page_html))
-    print("render_text_head=>", page_html)
-
-    page_html = dp_headless_html(target_url)
-    page_html = compress_html(page_html, only_text=True)
-    # print("render_text=>", compress_html(page_html))
-    print("render_text=>", page_html)
-
-    # print("\n")
-    # raw_html, status_code = requests_html(target_url)
-    # raw_html = compress_html(raw_html, only_text=True)
-    # # print("raw_html=>", status_code, compress_html(raw_html))
-    # print("raw_html=>", status_code, raw_html)
-    # print("\n")
-
-
-if __name__ == '__main__':
-    # raw_html, status_code = requests_html("https://www.nxzgh.org.cn/#/newsCenter/index2/2")
-    # raw_html, status_code = requests_html("http://www.ncha.gov.cn/col/col722/index.html")
-    # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
-    # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
-    # url = "https://www.nmpa.gov.cn/yaowen/ypjgyw/index.html"
-    # url = "http://www.customs.gov.cn/customs/xwfb34/302425/index.html"
-    # url = "https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html"
-    # url = "https://www.gjxfj.gov.cn/gjxfj/news/ttxw.htm"
-    # url = "https://www.gsgh.org.cn/#/moreNews_?position=%E7%9C%81%E6%80%BB%E6%96%B0%E9%97%BB&categoryId=502"
-    url = "https://www.acftu.org/xwdt/ghyw/"
-    # url = "https://www.nxzgh.org.cn/#/newsCenter/index2/2"  # 移动waf，会检测浏览器无头
-    # url = "https://www.chengdu.gov.cn/cdsrmzf/zfxx/cjj.shtml" # 超严格瑞数6，怀疑他会检测端口，我使用9222端口无论如何都获取不到结果
-    # url = "https://www.jsgh.org/col/col3577/index.html?uid=18462&pageNum=1"
-    # render_html = dp_headless_html(url)
-    # print("\n")
-    # print("render_html=>", render_html)
-    # for i in range(20):
-    test(url)
-    # todo: 大致盘一下各种判定的逻辑【以下的所有压缩比之间的差距均取“绝对值”】
-    #  1. 如果requests、无头、有头获取到的压缩比之间从差距都在15%以内，则认定该页面是静态页面，此时优先使用requests请求
-    #  2. 如果requests的status_code为特定的412，或者521，则判定是瑞数和jsl。[此时还有一个特点：requests的压缩比会与其他两种方式获取到的压缩比差距非常大(一两千的那种)]
-    #  3. 如果requests、无头、有头获取到的压缩比之间差距都在40%以上，则判定该页面只可以用有头采集
-    #  4. 如果无头和有头获取到的压缩比之间差距小于15%，但是requests和无头的差距大于40%，则认定该页面可以使用无头浏览器采集
-    #  5. 如果requests和有头获取到的压缩比之间差距小于15%，但是无头和有头的差距大于40%，则认定该页面优先使用有头浏览器采集
-    #  【此时可能是：1.使用了别的检测无头的waf。2.网站使用瑞数，但是这次请求没有拦截requests（不知道是不是瑞数那边故意设置的），
-    #   此时如果想进一步判定是否是瑞数，可以使用有头浏览器取一下cookies，如果cookies里面存在瑞数的cookie，那么就可以断定是瑞数】
-    #  6.
+    return result, compress_rate
+
+
+# 通过cookie判断是否有waf，需要通过遇到的例子，不断的完善cookie判别函数
+def assert_waf_cookie(cookies: list):
+    for cookie in cookies:
+        cookie_name = cookie['name']
+        cookie_value = cookie['value']
+        if len(cookie_name) == 13 and len(cookie_value) == 88:
+            return True, "瑞数"
+        if "_jsl" in cookie_name:
+            return True, "加速乐"
+    return False, "没有waf"
+
+
+# 对dp_mcp的消息打包
+def dp_mcp_message_pack(message: str, **kwargs):
+    text_obj = {key: value for key, value in kwargs.items()}
+    text_obj.update({"message": message})
+    return {
+        "content": [{
+            "type": "text",
+            # "text": json.dumps(text_obj, ensure_ascii=False)
+            "text": text_obj
+        }]
+    }
+
+# todo: 大致盘一下各种判定的逻辑【以下的所有压缩比之间的差距均取“绝对值”】
+#  1. 如果requests、无头、有头获取到的压缩比之间从差距都在15%以内，则认定该页面是静态页面，此时优先使用requests请求
+#  2. 如果requests的status_code为特定的412，或者521，则判定是瑞数和jsl。[此时还有一个特点：requests的压缩比会与其他两种方式获取到的压缩比差距非常大(一两千的那种)]
+#  3. 如果requests、无头、有头获取到的压缩比之间差距都在40%以上，则判定该页面只可以用有头采集
+#  4. 如果无头和有头获取到的压缩比之间差距小于15%，但是requests和无头的差距大于40%，则认定该页面可以使用无头浏览器采集
+#  5. 如果requests和有头获取到的压缩比之间差距小于15%，但是无头和有头的差距大于40%，则认定该页面优先使用有头浏览器采集
+#  【此时可能是：1.使用了别的检测无头的waf。2.网站使用瑞数，但是这次请求没有拦截requests（不知道是不是瑞数那边故意设置的），
+#   此时如果想进一步判定是否是瑞数，可以使用有头浏览器取一下cookies，如果cookies里面存在瑞数的cookie，那么就可以断定是瑞数】

jarvis_brain-0.1.3.dist-info/RECORD

@@ -1,11 +0,0 @@
-mcp_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mcp_tools/dp_tools.py,sha256=jDilSFnOdwGC2453983k6VdGHNu3lDaDcioa-XXhUc0,9003
-mcp_tools/main.py,sha256=lxH8PafR4KVM0-OURgHD0jNaC68kLmPYaNJ1wSKfNdY,654
-mcp_tools/requests_tools.py,sha256=2rP4NBYaGTJI9-Txn2vaoPH5A2fWM05RNNjiSCHvbtk,1373
-tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tools/simhash_tools.py,sha256=eRxUqcWJiNMNZPpVRBk1kI7OYSK_vThoVg3jctFfmhY,7368
-tools/tools.py,sha256=AP5lw65o2VYQ0Q6hEvCRA7m9WD6e7G8j4KBkdheLNmU,6913
-jarvis_brain-0.1.3.dist-info/METADATA,sha256=wVy9AR3LB40xokpRF5U9MrsZxCJJ0DuFSEW4CUJ3ox8,275
-jarvis_brain-0.1.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-jarvis_brain-0.1.3.dist-info/entry_points.txt,sha256=YFQT4xpkUqt5dM5wlKPQQOqcjMuFrT9iuRAzIpAyH7U,51
-jarvis_brain-0.1.3.dist-info/RECORD,,

mcp_tools/requests_tools.py

@@ -1,40 +0,0 @@
-import json
-
-from fastmcp import FastMCP
-from tools.tools import requests_html, dp_headless_html
-
-
-def register_assert_waf(mcp: FastMCP):
-    @mcp.tool(name="assert_waf", description="判断传入的url对应的网页是否存在瑞数、jsl等风控防火墙")
-    def assert_waf(url: str):
-        # 先试用requests判断瑞数和jsl
-        text, code = requests_html(url)
-        waf_text_type = {
-            521: "jsl",
-            412: "瑞数",
-            "触发WAF": "其他waf"
-        }
-        has_waf = code in waf_text_type.keys()
-        if not has_waf:
-            # 不是瑞数和jsl，使用无头浏览器判断是否为其他waf
-            headless_html = dp_headless_html(url)
-            if "触发WAF" in headless_html:
-                has_waf = True
-                code = "触发WAF"
-        if not has_waf:
-            waf_type = "不存在waf"
-        else:
-            waf_type = waf_text_type[code]
-        return {
-            "content": [{
-                "type": "text",
-                "text": json.dumps(
-                    {
-                        "message": f"链接{url} [{'存在' if has_waf else '不存在'}] waf",
-                        "url": url,
-                        "waf_type": waf_type,
-                        "has_waf": has_waf
-                    }, ensure_ascii=False
-                )
-            }]
-        }

tools/simhash_tools.py

@@ -1,228 +0,0 @@
-import re
-from bs4 import BeautifulSoup
-from simhash import Simhash, SimhashIndex
-import jieba
-import hashlib
-from tools.tools import compress_html
-
-
-class HTMLSimHashComparator:
-    def __init__(self, html1, html2, is_file=False):
-        """
-        初始化比较器
-
-        Args:
-            html1: 第一个HTML内容或文件路径
-            html2: 第二个HTML内容或文件路径
-            is_file: 是否为文件路径（True则为文件路径，False则为HTML字符串）
-        """
-        if is_file:
-            with open(html1, 'r', encoding='utf-8') as f1:
-                self.html1 = f1.read()
-            with open(html2, 'r', encoding='utf-8') as f2:
-                self.html2 = f2.read()
-        else:
-            self.html1 = html1
-            self.html2 = html2
-
-    def clean_html(self, html_content):
-        text = compress_html(html_content, only_text=True)
-        return text
-
-    def extract_features(self, text):
-        """从文本中提取特征"""
-        # 使用jieba进行中文分词（如果是中文内容）
-        # 如果是英文，可以使用空格分词或其他方式
-        words = jieba.lcut(text)
-
-        # 过滤停用词和短词
-        stop_words = set(['的', '了', '在', '是', '我', '有', '和', '就',
-                          '不', '人', '都', '一', '一个', '上', '也', '很',
-                          '到', '说', '要', '去', '你', '会', '着', '没有',
-                          '看', '好', '自己', '这'])
-
-        features = []
-        for word in words:
-            # 过滤停用词和长度小于2的词
-            if word not in stop_words and len(word) >= 2:
-                features.append(word)
-
-        return features
-
-    def calculate_simhash(self, html_content):
-        """计算HTML内容的SimHash值"""
-        # 清洗HTML
-        text = self.clean_html(html_content)
-
-        # 提取特征
-        features = self.extract_features(text)
-
-        # 计算SimHash（默认使用64位）
-        simhash = Simhash(features, f=64)
-
-        return simhash
-
-    def compare_simhash(self):
-        """比较两个HTML的SimHash值"""
-        # 计算SimHash值
-        simhash1 = self.calculate_simhash(self.html1)
-        simhash2 = self.calculate_simhash(self.html2)
-
-        # 计算汉明距离
-        hamming_distance = simhash1.distance(simhash2)
-
-        # 计算相似度（0-1之间）
-        # 64位SimHash的最大汉明距离是64
-        similarity = 1 - (hamming_distance / 64)
-
-        return {
-            'simhash1': bin(simhash1.value),
-            'simhash2': bin(simhash2.value),
-            'hamming_distance': hamming_distance,
-            'similarity': similarity,
-            'similarity_percentage': f"{similarity * 100:.2f}%"
-        }
-
-    def compare_with_threshold(self, threshold=0.8):
-        """基于阈值判断是否相似"""
-        result = self.compare_simhash()
-        is_similar = result['similarity'] >= threshold
-
-        return {
-            **result,
-            'threshold': threshold,
-            'is_similar': is_similar
-        }
-
-
-# 3. 使用示例
-def main():
-    # 示例1：直接传入HTML字符串
-    html1 = """
-    <html>
-        <head><title>测试页面1</title></head>
-        <body>
-            <h1>欢迎来到我的网站</h1>
-            <p>这是一个测试页面，用于演示SimHash比较。</p>
-            <p>Python是一种流行的编程语言。</p>
-        </body>
-    </html>
-    """
-
-    html2 = """
-    <html>
-        <head><title>测试页面2</title></head>
-        <body>
-            <h1>欢迎访问我的网站</h1>
-            <p>这是一个测试页面，用于展示SimHash比较功能。</p>
-            <p>Python编程语言非常流行。</p>
-        </body>
-    </html>
-    """
-
-    # 创建比较器并比较
-    comparator = HTMLSimHashComparator(html1, html2)
-    result = comparator.compare_simhash()
-
-    print("SimHash比较结果:")
-    print(f"页面1 SimHash: {result['simhash1']}")
-    print(f"页面2 SimHash: {result['simhash2']}")
-    print(f"汉明距离: {result['hamming_distance']}")
-    print(f"相似度: {result['similarity_percentage']}")
-
-    # 基于阈值判断
-    threshold_result = comparator.compare_with_threshold(threshold=0.7)
-    print(f"\n基于阈值{threshold_result['threshold']}的判断:")
-    print(f"是否相似: {threshold_result['is_similar']}")
-
-
-# 4. 高级功能：批量比较和聚类
-class BatchHTMLComparator:
-    def __init__(self):
-        self.documents = []
-        self.simhashes = []
-
-    def add_document(self, doc_id, html_content):
-        """添加文档到比较器"""
-        comparator = HTMLSimHashComparator(html_content, html_content)
-        simhash = comparator.calculate_simhash(html_content)
-
-        self.documents.append({
-            'id': doc_id,
-            'content': html_content,
-            'simhash': simhash
-        })
-        self.simhashes.append((doc_id, simhash))
-
-    def find_duplicates(self, k=3):
-        """查找相似的文档（k为汉明距离阈值）"""
-        index = SimhashIndex(self.simhashes, k=k)
-
-        duplicates = []
-        for doc_id, simhash in self.simhashes:
-            # 查找相似文档
-            similar_ids = index.get_near_dups(simhash)
-            if len(similar_ids) > 1:
-                duplicates.append({
-                    'doc_id': doc_id,
-                    'similar_docs': similar_ids
-                })
-
-        return duplicates
-
-    def compare_all_pairs(self):
-        """比较所有文档对"""
-        comparisons = []
-        n = len(self.documents)
-
-        for i in range(n):
-            for j in range(i + 1, n):
-                comparator = HTMLSimHashComparator(
-                    self.documents[i]['content'],
-                    self.documents[j]['content']
-                )
-                result = comparator.compare_simhash()
-
-                comparisons.append({
-                    'doc1': self.documents[i]['id'],
-                    'doc2': self.documents[j]['id'],
-                    'hamming_distance': result['hamming_distance'],
-                    'similarity': result['similarity']
-                })
-
-        return sorted(comparisons, key=lambda x: x['similarity'], reverse=True)
-
-
-if __name__ == "__main__":
-    # 运行示例
-    main()
-
-    # 批量比较示例
-    print("\n=== 批量比较示例 ===")
-    batch_comparator = BatchHTMLComparator()
-
-    # 添加多个文档
-    batch_comparator.add_document("doc1", """
-        <html><body><h1>Python编程</h1><p>学习Python很有趣。</p></body></html>
-    """)
-
-    batch_comparator.add_document("doc2", """
-        <html><body><h1>Python代码</h1><p>编写Python代码很有趣。</p></body></html>
-    """)
-
-    batch_comparator.add_document("doc3", """
-        <html><body><h1>Java编程</h1><p>Java是一种编程语言。</p></body></html>
-    """)
-
-    # 查找重复文档
-    duplicates = batch_comparator.find_duplicates(k=3)
-    print("相似的文档:")
-    for dup in duplicates:
-        print(f"文档 {dup['doc_id']} 与 {dup['similar_docs']} 相似")
-
-    # 比较所有文档对
-    all_comparisons = batch_comparator.compare_all_pairs()
-    print("\n所有文档对比较:")
-    for comp in all_comparisons:
-        print(f"{comp['doc1']} vs {comp['doc2']}: "
-              f"相似度 {comp['similarity']:.2%}")