Jarvis-Brain 0.1.0__py3-none-any.whl → 0.1.2__py3-none-any.whl

{jarvis_brain-0.1.0.dist-info → jarvis_brain-0.1.2.dist-info}/METADATA

@@ -1,8 +1,11 @@
 Metadata-Version: 2.4
 Name: Jarvis_Brain
-Version: 0.1.0
+Version: 0.1.2
 Summary: Jarvis brain
 Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4
 Requires-Dist: drissionpage
 Requires-Dist: fastmcp
 Requires-Dist: htmlmin
+Requires-Dist: jieba
+Requires-Dist: simhash

jarvis_brain-0.1.2.dist-info/RECORD

@@ -0,0 +1,11 @@
+mcp_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mcp_tools/dp_tools.py,sha256=xtpJoRmpqZQRjfVUEIjo3KAd5_SqK01J1VAt8U79xbA,8448
+mcp_tools/main.py,sha256=lxH8PafR4KVM0-OURgHD0jNaC68kLmPYaNJ1wSKfNdY,654
+mcp_tools/requests_tools.py,sha256=K1eHbvhzWhLzn9AZhjDGTmDu_yipvkEM9IOhveDMsPM,1063
+tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tools/simhash_tools.py,sha256=eRxUqcWJiNMNZPpVRBk1kI7OYSK_vThoVg3jctFfmhY,7368
+tools/tools.py,sha256=lPlXdVA4Dx3irP6alMOgmE-HxxuN_y4XQ3Q-fZyV-OU,3482
+jarvis_brain-0.1.2.dist-info/METADATA,sha256=Otgmq6QJe4XoWiIhC1i6jBpQsgWhraoPpi8v8Su7SYw,250
+jarvis_brain-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+jarvis_brain-0.1.2.dist-info/entry_points.txt,sha256=YFQT4xpkUqt5dM5wlKPQQOqcjMuFrT9iuRAzIpAyH7U,51
+jarvis_brain-0.1.2.dist-info/RECORD,,

mcp_tools/dp_tools.py

@@ -1,45 +1,20 @@
 import json
 import os
 import random
+from pyexpat.errors import messages
 from typing import Any
 
-import htmlmin
 from DrissionPage import ChromiumPage, ChromiumOptions
 from DrissionPage._elements.none_element import NoneElement
 from fastmcp import FastMCP
-from lxml import html, etree
+
+from tools.tools import compress_html, requests_html
+from tools.simhash_tools import HTMLSimHashComparator
 
 html_source_code_local_save_path = os.path.join(os.getcwd(), "html-source-code")
 browser_pool = {}
 
 
-# 压缩html
-def compress_html(content):
-    doc = html.fromstring(content)
-    # 删除 style 和 script 标签
-    for element in doc.xpath('//style | //script'):
-        element.getparent().remove(element)
-
-    # 删除 link 标签
-    for link in doc.xpath('//link[@rel="stylesheet"]'):
-        link.getparent().remove(link)
-
-    # 删除 style 属性
-    for element in doc.xpath('//*[@style]'):
-        element.attrib.pop('style')
-
-    # 删除所有 on* 事件属性
-    for element in doc.xpath('//*'):
-        for attr in list(element.attrib.keys()):
-            if attr.startswith('on'):
-                element.attrib.pop(attr)
-
-    result = etree.tostring(doc, encoding='unicode')
-    result = htmlmin.minify((result))
-    print(f"html压缩比=> {len(content) / len(result) * 100:.2f}%")
-    return result
-
-
 # 随机一个浏览器池中不存在的端口，创建一个浏览器，返回随机端口，和浏览器对象。
 def create_browser():
     global browser_pool
@@ -56,6 +31,12 @@ def get_page(port):
     return browser_pool.get(port, None)
 
 
+# 根据传入的端口查找并弹出一个浏览器对象
+def remove_page(port):
+    browser = browser_pool.pop(port, None)
+    return browser is not None, browser
+
+
 def register_visit_url(mcp: FastMCP):
     @mcp.tool(name="visit_url", description="使用Drissionpage打开url访问某个网站")
     async def visit_url(url: str) -> dict[str, Any]:
@@ -81,6 +62,8 @@ def register_get_html(mcp: FastMCP):
         _browser = get_page(browser_port)
         tab = _browser.get_tab(tab_id)
         file_name = tab.title + f"_{tab_id}.html"
+        if not os.path.exists(html_source_code_local_save_path):
+            os.makedirs(html_source_code_local_save_path)
         abs_path = os.path.join(html_source_code_local_save_path, file_name)
         with open(abs_path, "w", encoding="utf-8") as f:
             f.write(compress_html(tab.html))
@@ -165,9 +148,64 @@ def register_check_selector(mcp: FastMCP):
             }]
         }
 
-# def main():
-#     mcp.run()
-#
-#
-# if __name__ == '__main__':
-#     main()
+
+def register_quit_browser(mcp: FastMCP):
+    @mcp.tool(name="quit_browser", description="退出浏览器会话，关闭浏览器")
+    async def quit_browser(browser_port: int) -> dict[str, Any]:
+        flag, _browser = remove_page(browser_port)
+        if flag:
+            _browser.quit()
+        return {
+            "content": [{
+                "type": "text",
+                "text": json.dumps({
+                    "message": f"浏览器[{browser_port}]，退出会话，关闭浏览器{'成功' if flag else '失败'}",
+                    "browser_port": browser_port,
+                    "quit_flag": flag,
+                }, ensure_ascii=False)
+            }]
+        }
+
+
+def register_assert_Static_Web(mcp: FastMCP):
+    @mcp.tool(name="assert_Static_Web", description="判断tab页中的网页是否是静态网页")
+    def assert_Static_Web(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = get_page(browser_port)
+        target_tab = _browser.get_tab(tab_id)
+        target_url = target_tab.url
+        raw_html, stat_code = requests_html(target_url)
+        if stat_code != 200:
+            return {
+                "content": [{
+                    "type": "text",
+                    "text": json.dumps({
+                        "messages": f"已完成tab页:【{tab_id}】的分析，该tab页使用requests获取的html状态码为[{stat_code}]，并非200，请使用assert_waf判断是否有waf",
+                        "tab_id": tab_id,
+                        "url": target_url,
+                        "stat_code": stat_code,
+                    }, ensure_ascii=False)
+                }]
+            }
+        render_html = target_tab.html
+        comparator = HTMLSimHashComparator(raw_html, render_html, False)
+        result = comparator.compare_simhash()
+        print("SimHash比较结果:")
+        print(f"页面1 SimHash: {result['simhash1']}")
+        print(f"页面2 SimHash: {result['simhash2']}")
+        print(f"汉明距离: {result['hamming_distance']}")
+        print(f"相似度: {result['similarity_percentage']}")
+        threshold_result = comparator.compare_with_threshold(threshold=0.7)
+        print(f"\n基于阈值{threshold_result['threshold']}的判断:")
+        print(f"是否相似: {threshold_result['is_similar']}")
+        static_html_flag = threshold_result['is_similar']
+        return {
+            "content": [{
+                "type": "text",
+                "text": json.dumps({
+                    "message": f"已完成tab页:【{tab_id}】的分析，该tab页静态页面和渲染页面的相似度为[{result['similarity_percentage']}]，判定为 {'静态页面' if static_html_flag else '动态渲染'}",
+                    "tab_id": tab_id,
+                    "is_static_web": static_html_flag,
+                    "static_web_possibility": result['similarity_percentage'],
+                }, ensure_ascii=False)
+            }]
+        }

mcp_tools/main.py

@@ -1,13 +1,14 @@
 # main.py
 from mcp_tools.dp_tools import *
+from mcp_tools.requests_tools import *
 from fastmcp import FastMCP
 
-mcp = FastMCP("Jarvis Brain DrissionPage mcp")
+mcp = FastMCP("Jarvis Brain Mcp Tools")
 
 # 根据环境变量加载模块
-enabled_modules = os.getenv("MCP_MODULES", "DrissionPage").split(",")
+enabled_modules = os.getenv("MCP_MODULES", "TeamNode-Dp").split(",")
 
-if "DrissionPage" in enabled_modules:
+if "TeamNode-Dp" in enabled_modules:
     register_visit_url(mcp)
     register_close_tab(mcp)
     register_switch_tab(mcp)
@@ -15,6 +16,14 @@ if "DrissionPage" in enabled_modules:
     register_get_new_tab(mcp)
     register_check_selector(mcp)
 
+if "JarvisNode" in enabled_modules:
+    register_assert_waf(mcp)
+    register_assert_Static_Web(mcp)
+
 
 def main():
-    mcp.run()
+    mcp.run(transport="stdio")
+
+
+if __name__ == '__main__':
+    main()

mcp_tools/requests_tools.py

@@ -0,0 +1,36 @@
+import json
+
+from fastmcp import FastMCP
+from tools.tools import requests_html
+from DrissionPage import ChromiumPage, ChromiumOptions
+
+
+def register_assert_waf(mcp: FastMCP):
+    @mcp.tool(name="assert_waf", description="判断传入的url对应的网页是否存在瑞数、jsl等风控防火墙")
+    def assert_waf(url: str):
+        text, code = requests_html(url)
+        waf_text_type = {
+            521: "jsl",
+            412: "瑞数"
+        }
+        has_waf = code in waf_text_type.keys()
+        if not has_waf:
+            waf_type = "不存在waf"
+        else:
+            waf_type = waf_text_type[code]
+        return {
+            "content": [{
+                "type": "text",
+                "text": json.dumps(
+                    {
+                        "message": f"链接{url} [{'存在' if has_waf else '不存在'}] waf",
+                        "url": url,
+                        "waf_type": waf_type,
+                        "has_waf": has_waf
+                    }, ensure_ascii=False
+                )
+            }]
+        }
+
+
+

tools/simhash_tools.py

@@ -0,0 +1,228 @@
+import re
+from bs4 import BeautifulSoup
+from simhash import Simhash, SimhashIndex
+import jieba
+import hashlib
+from tools.tools import compress_html
+
+
+class HTMLSimHashComparator:
+    def __init__(self, html1, html2, is_file=False):
+        """
+        初始化比较器
+
+        Args:
+            html1: 第一个HTML内容或文件路径
+            html2: 第二个HTML内容或文件路径
+            is_file: 是否为文件路径（True则为文件路径，False则为HTML字符串）
+        """
+        if is_file:
+            with open(html1, 'r', encoding='utf-8') as f1:
+                self.html1 = f1.read()
+            with open(html2, 'r', encoding='utf-8') as f2:
+                self.html2 = f2.read()
+        else:
+            self.html1 = html1
+            self.html2 = html2
+
+    def clean_html(self, html_content):
+        text = compress_html(html_content, only_text=True)
+        return text
+
+    def extract_features(self, text):
+        """从文本中提取特征"""
+        # 使用jieba进行中文分词（如果是中文内容）
+        # 如果是英文，可以使用空格分词或其他方式
+        words = jieba.lcut(text)
+
+        # 过滤停用词和短词
+        stop_words = set(['的', '了', '在', '是', '我', '有', '和', '就',
+                          '不', '人', '都', '一', '一个', '上', '也', '很',
+                          '到', '说', '要', '去', '你', '会', '着', '没有',
+                          '看', '好', '自己', '这'])
+
+        features = []
+        for word in words:
+            # 过滤停用词和长度小于2的词
+            if word not in stop_words and len(word) >= 2:
+                features.append(word)
+
+        return features
+
+    def calculate_simhash(self, html_content):
+        """计算HTML内容的SimHash值"""
+        # 清洗HTML
+        text = self.clean_html(html_content)
+
+        # 提取特征
+        features = self.extract_features(text)
+
+        # 计算SimHash（默认使用64位）
+        simhash = Simhash(features, f=64)
+
+        return simhash
+
+    def compare_simhash(self):
+        """比较两个HTML的SimHash值"""
+        # 计算SimHash值
+        simhash1 = self.calculate_simhash(self.html1)
+        simhash2 = self.calculate_simhash(self.html2)
+
+        # 计算汉明距离
+        hamming_distance = simhash1.distance(simhash2)
+
+        # 计算相似度（0-1之间）
+        # 64位SimHash的最大汉明距离是64
+        similarity = 1 - (hamming_distance / 64)
+
+        return {
+            'simhash1': bin(simhash1.value),
+            'simhash2': bin(simhash2.value),
+            'hamming_distance': hamming_distance,
+            'similarity': similarity,
+            'similarity_percentage': f"{similarity * 100:.2f}%"
+        }
+
+    def compare_with_threshold(self, threshold=0.8):
+        """基于阈值判断是否相似"""
+        result = self.compare_simhash()
+        is_similar = result['similarity'] >= threshold
+
+        return {
+            **result,
+            'threshold': threshold,
+            'is_similar': is_similar
+        }
+
+
+# 3. 使用示例
+def main():
+    # 示例1：直接传入HTML字符串
+    html1 = """
+    <html>
+        <head><title>测试页面1</title></head>
+        <body>
+            <h1>欢迎来到我的网站</h1>
+            <p>这是一个测试页面，用于演示SimHash比较。</p>
+            <p>Python是一种流行的编程语言。</p>
+        </body>
+    </html>
+    """
+
+    html2 = """
+    <html>
+        <head><title>测试页面2</title></head>
+        <body>
+            <h1>欢迎访问我的网站</h1>
+            <p>这是一个测试页面，用于展示SimHash比较功能。</p>
+            <p>Python编程语言非常流行。</p>
+        </body>
+    </html>
+    """
+
+    # 创建比较器并比较
+    comparator = HTMLSimHashComparator(html1, html2)
+    result = comparator.compare_simhash()
+
+    print("SimHash比较结果:")
+    print(f"页面1 SimHash: {result['simhash1']}")
+    print(f"页面2 SimHash: {result['simhash2']}")
+    print(f"汉明距离: {result['hamming_distance']}")
+    print(f"相似度: {result['similarity_percentage']}")
+
+    # 基于阈值判断
+    threshold_result = comparator.compare_with_threshold(threshold=0.7)
+    print(f"\n基于阈值{threshold_result['threshold']}的判断:")
+    print(f"是否相似: {threshold_result['is_similar']}")
+
+
+# 4. 高级功能：批量比较和聚类
+class BatchHTMLComparator:
+    def __init__(self):
+        self.documents = []
+        self.simhashes = []
+
+    def add_document(self, doc_id, html_content):
+        """添加文档到比较器"""
+        comparator = HTMLSimHashComparator(html_content, html_content)
+        simhash = comparator.calculate_simhash(html_content)
+
+        self.documents.append({
+            'id': doc_id,
+            'content': html_content,
+            'simhash': simhash
+        })
+        self.simhashes.append((doc_id, simhash))
+
+    def find_duplicates(self, k=3):
+        """查找相似的文档（k为汉明距离阈值）"""
+        index = SimhashIndex(self.simhashes, k=k)
+
+        duplicates = []
+        for doc_id, simhash in self.simhashes:
+            # 查找相似文档
+            similar_ids = index.get_near_dups(simhash)
+            if len(similar_ids) > 1:
+                duplicates.append({
+                    'doc_id': doc_id,
+                    'similar_docs': similar_ids
+                })
+
+        return duplicates
+
+    def compare_all_pairs(self):
+        """比较所有文档对"""
+        comparisons = []
+        n = len(self.documents)
+
+        for i in range(n):
+            for j in range(i + 1, n):
+                comparator = HTMLSimHashComparator(
+                    self.documents[i]['content'],
+                    self.documents[j]['content']
+                )
+                result = comparator.compare_simhash()
+
+                comparisons.append({
+                    'doc1': self.documents[i]['id'],
+                    'doc2': self.documents[j]['id'],
+                    'hamming_distance': result['hamming_distance'],
+                    'similarity': result['similarity']
+                })
+
+        return sorted(comparisons, key=lambda x: x['similarity'], reverse=True)
+
+
+if __name__ == "__main__":
+    # 运行示例
+    main()
+
+    # 批量比较示例
+    print("\n=== 批量比较示例 ===")
+    batch_comparator = BatchHTMLComparator()
+
+    # 添加多个文档
+    batch_comparator.add_document("doc1", """
+        <html><body><h1>Python编程</h1><p>学习Python很有趣。</p></body></html>
+    """)
+
+    batch_comparator.add_document("doc2", """
+        <html><body><h1>Python代码</h1><p>编写Python代码很有趣。</p></body></html>
+    """)
+
+    batch_comparator.add_document("doc3", """
+        <html><body><h1>Java编程</h1><p>Java是一种编程语言。</p></body></html>
+    """)
+
+    # 查找重复文档
+    duplicates = batch_comparator.find_duplicates(k=3)
+    print("相似的文档:")
+    for dup in duplicates:
+        print(f"文档 {dup['doc_id']} 与 {dup['similar_docs']} 相似")
+
+    # 比较所有文档对
+    all_comparisons = batch_comparator.compare_all_pairs()
+    print("\n所有文档对比较:")
+    for comp in all_comparisons:
+        print(f"{comp['doc1']} vs {comp['doc2']}: "
+              f"相似度 {comp['similarity']:.2%}")

tools/tools.py

@@ -0,0 +1,92 @@
+import htmlmin
+import requests
+from lxml import html, etree
+from bs4 import BeautifulSoup
+from DrissionPage import ChromiumPage, ChromiumOptions
+
+
+# 使用requests获取html，用于测试是否使用了瑞数和jsl
+def requests_html(url):
+    headers = {
+        # "sec-ch-ua": "\"Chromium\";v=\"142\", \"Google Chrome\";v=\"142\", \"Not_A Brand\";v=\"99\"",
+        # "sec-ch-ua-mobile": "?0",
+        # "Upgrade-Insecure-Requests": "1",
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+        # "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+        # "Sec-Fetch-Site": "none",
+        # "Sec-Fetch-Mode": "navigate",
+        # "Sec-Fetch-User": "?1",
+        # "Sec-Fetch-Dest": "document",
+        # "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"
+    }
+    response = requests.get(url, headers=headers, verify=False)
+    print("response headers=> ", response.headers)
+    return response.text, response.status_code
+
+
+# 使用dp无头模式获取html，用于测试是否使用了其他waf，如移动waf
+def dp_headless_html(url):
+    opt = ChromiumOptions().headless(True)
+    opt.set_argument('--no-sandbox')
+    page = ChromiumPage(opt)
+    page.get(url)
+    page_html = page.html
+    page.quit()
+    return page_html
+
+
+# 压缩html
+def compress_html(content, only_text=False):
+    doc = html.fromstring(content)
+    # 删除 style 和 script 标签
+    for element in doc.xpath('//style | //script'):
+        element.getparent().remove(element)
+
+    # 删除 link 标签
+    for link in doc.xpath('//link[@rel="stylesheet"]'):
+        link.getparent().remove(link)
+
+    # 删除 meta 标签（新增功能）
+    for meta in doc.xpath('//meta'):
+        meta.getparent().remove(meta)
+
+    # 删除 style 属性
+    for element in doc.xpath('//*[@style]'):
+        element.attrib.pop('style')
+
+    # 删除所有 on* 事件属性
+    for element in doc.xpath('//*'):
+        for attr in list(element.attrib.keys()):
+            if attr.startswith('on'):
+                element.attrib.pop(attr)
+
+    result = etree.tostring(doc, encoding='unicode')
+    result = htmlmin.minify(result)
+    print(f"html压缩比=> {len(content) / len(result) * 100:.2f}%")
+    if not only_text:
+        return result
+    soup = BeautifulSoup(result, 'html.parser')
+    result = soup.get_text(strip=True)
+    return result
+
+
+def test(target_url):
+    page_html = dp_headless_html(target_url)
+    print("render_text=>", compress_html(page_html, only_text=True))
+    print("\n")
+    raw_html, status_code = requests_html(target_url)
+    print("raw_html=>", status_code, compress_html(raw_html, only_text=True))
+    print("\n")
+
+
+if __name__ == '__main__':
+    # raw_html, status_code = requests_html("https://www.nxzgh.org.cn/#/newsCenter/index2/2")
+    # raw_html, status_code = requests_html("http://www.ncha.gov.cn/col/col722/index.html")
+    # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
+    # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
+    # url = "https://www.nmpa.gov.cn/yaowen/ypjgyw/index.html"
+    url = "http://www.customs.gov.cn/customs/xwfb34/302425/index.html"
+    # url = "https://www.acftu.org/xwdt/ghyw/"
+
+    for i in range(20):
+        test(url)

jarvis_brain-0.1.0.dist-info/RECORD

@@ -1,7 +0,0 @@
-mcp_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mcp_tools/dp_tools.py,sha256=f5jl6BHfuvqVjFplnUEvlzLbK0k0bJnMpR7cVHxWsy8,6017
-mcp_tools/main.py,sha256=mKuVyhAKAAKgbP3YSDVm5qehVNb5dtK8ybDZGQS8Y7U,465
-jarvis_brain-0.1.0.dist-info/METADATA,sha256=jBepatGeCD2ByZojTKYPyrH9Yg_Brtzj8mmbZqreu-c,176
-jarvis_brain-0.1.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-jarvis_brain-0.1.0.dist-info/entry_points.txt,sha256=YFQT4xpkUqt5dM5wlKPQQOqcjMuFrT9iuRAzIpAyH7U,51
-jarvis_brain-0.1.0.dist-info/RECORD,,