Jarvis-Brain 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

{jarvis_brain-0.1.2.dist-info → jarvis_brain-0.1.3.dist-info}/METADATA

@@ -1,9 +1,10 @@
 Metadata-Version: 2.4
 Name: Jarvis_Brain
-Version: 0.1.2
+Version: 0.1.3
 Summary: Jarvis brain
 Requires-Python: >=3.10
 Requires-Dist: beautifulsoup4
+Requires-Dist: curl-cffi
 Requires-Dist: drissionpage
 Requires-Dist: fastmcp
 Requires-Dist: htmlmin

jarvis_brain-0.1.3.dist-info/RECORD

@@ -0,0 +1,11 @@
+mcp_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+mcp_tools/dp_tools.py,sha256=jDilSFnOdwGC2453983k6VdGHNu3lDaDcioa-XXhUc0,9003
+mcp_tools/main.py,sha256=lxH8PafR4KVM0-OURgHD0jNaC68kLmPYaNJ1wSKfNdY,654
+mcp_tools/requests_tools.py,sha256=2rP4NBYaGTJI9-Txn2vaoPH5A2fWM05RNNjiSCHvbtk,1373
+tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tools/simhash_tools.py,sha256=eRxUqcWJiNMNZPpVRBk1kI7OYSK_vThoVg3jctFfmhY,7368
+tools/tools.py,sha256=AP5lw65o2VYQ0Q6hEvCRA7m9WD6e7G8j4KBkdheLNmU,6913
+jarvis_brain-0.1.3.dist-info/METADATA,sha256=wVy9AR3LB40xokpRF5U9MrsZxCJJ0DuFSEW4CUJ3ox8,275
+jarvis_brain-0.1.3.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
+jarvis_brain-0.1.3.dist-info/entry_points.txt,sha256=YFQT4xpkUqt5dM5wlKPQQOqcjMuFrT9iuRAzIpAyH7U,51
+jarvis_brain-0.1.3.dist-info/RECORD,,

mcp_tools/dp_tools.py

@@ -1,9 +1,9 @@
 import json
 import os
 import random
-from pyexpat.errors import messages
 from typing import Any
 
+from urllib.parse import urlparse
 from DrissionPage import ChromiumPage, ChromiumOptions
 from DrissionPage._elements.none_element import NoneElement
 from fastmcp import FastMCP
@@ -209,3 +209,23 @@ def register_assert_Static_Web(mcp: FastMCP):
                 }, ensure_ascii=False)
             }]
         }
+
+
+def register_get_cookies(mcp: FastMCP):
+    @mcp.tool(name="get_cookies")
+    def get_cookies(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = get_page(browser_port)
+        tab = _browser.get_tab(tab_id)
+        target_url = tab.url
+        parser = urlparse(target_url)
+        domain = parser.netloc
+        cookies = tab.cookies()
+        # tab.cookies
+        return {
+            "content": [{
+                "type": "text",
+                "text": json.dumps({
+
+                }, ensure_ascii=False)
+            }]
+        }

mcp_tools/requests_tools.py

@@ -1,19 +1,26 @@
 import json
 
 from fastmcp import FastMCP
-from tools.tools import requests_html
-from DrissionPage import ChromiumPage, ChromiumOptions
+from tools.tools import requests_html, dp_headless_html
 
 
 def register_assert_waf(mcp: FastMCP):
     @mcp.tool(name="assert_waf", description="判断传入的url对应的网页是否存在瑞数、jsl等风控防火墙")
     def assert_waf(url: str):
+        # 先试用requests判断瑞数和jsl
         text, code = requests_html(url)
         waf_text_type = {
             521: "jsl",
-            412: "瑞数"
+            412: "瑞数",
+            "触发WAF": "其他waf"
         }
         has_waf = code in waf_text_type.keys()
+        if not has_waf:
+            # 不是瑞数和jsl，使用无头浏览器判断是否为其他waf
+            headless_html = dp_headless_html(url)
+            if "触发WAF" in headless_html:
+                has_waf = True
+                code = "触发WAF"
         if not has_waf:
             waf_type = "不存在waf"
         else:
@@ -31,6 +38,3 @@ def register_assert_waf(mcp: FastMCP):
                 )
             }]
         }
-
-
-

tools/tools.py

@@ -1,26 +1,33 @@
+import time
+
 import htmlmin
-import requests
+from curl_cffi import requests
 from lxml import html, etree
 from bs4 import BeautifulSoup
 from DrissionPage import ChromiumPage, ChromiumOptions
 
 
+# 传入requests的set-cookie的str，返回一个cookie dict
+def cookie_str2dict(cookie_str: str):
+    if cookie_str == "":
+        return {}
+    cookie_dict = {}
+    cookie_list = [cookie.split(";")[0] for cookie in cookie_str.split("HttpOnly,")]
+    for cookie in cookie_list:
+        key, value = cookie.split("=")
+        cookie_dict[key] = value
+    return cookie_dict
+
+
 # 使用requests获取html，用于测试是否使用了瑞数和jsl
 def requests_html(url):
     headers = {
-        # "sec-ch-ua": "\"Chromium\";v=\"142\", \"Google Chrome\";v=\"142\", \"Not_A Brand\";v=\"99\"",
-        # "sec-ch-ua-mobile": "?0",
-        # "Upgrade-Insecure-Requests": "1",
         "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
-        # "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-        # "Sec-Fetch-Site": "none",
-        # "Sec-Fetch-Mode": "navigate",
-        # "Sec-Fetch-User": "?1",
-        # "Sec-Fetch-Dest": "document",
-        # "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"
     }
     response = requests.get(url, headers=headers, verify=False)
-    print("response headers=> ", response.headers)
+    # print("response headers=> ", type(response.headers.get("Set-Cookie")),
+    #       cookie_str2dict(response.headers.get("Set-Cookie", "")))
+    response.encoding = "utf-8"
     return response.text, response.status_code
 
 
@@ -28,9 +35,42 @@ def requests_html(url):
 def dp_headless_html(url):
     opt = ChromiumOptions().headless(True)
     opt.set_argument('--no-sandbox')
+    opt.set_local_port(9976)
+
     page = ChromiumPage(opt)
-    page.get(url)
-    page_html = page.html
+    # page = ChromiumPage()
+
+    tab = page.latest_tab
+    # tab.set.load_mode.normal()
+    tab.get(url)
+    # tab.wait.eles_loaded()
+    time.sleep(10)
+    # tab.wait.load_start()
+    page_html = tab.html
+    # print(page_html)
+    # print("dp_cookies=>", tab.cookies())
+    page.quit()
+    return page_html
+
+
+# 使用dp无头模式获取html，用于测试是否使用了其他waf，如移动waf
+def dp_html(url):
+    opt = ChromiumOptions()
+    opt.set_local_port(9223)
+    # opt.set_argument('--no-sandbox')
+    page = ChromiumPage(opt)
+    # page = ChromiumPage()
+
+    tab = page.latest_tab
+    # tab.set.load_mode.normal()
+    tab.get(url)
+    # tab.wait.eles_loaded()
+    # time.sleep(10)
+
+    tab.wait.doc_loaded()
+    page_html = tab.html
+    # print(page_html)
+    # print("dp_cookies=>", tab.cookies())
     page.quit()
     return page_html
 
@@ -71,12 +111,26 @@ def compress_html(content, only_text=False):
 
 
 def test(target_url):
-    page_html = dp_headless_html(target_url)
-    print("render_text=>", compress_html(page_html, only_text=True))
-    print("\n")
     raw_html, status_code = requests_html(target_url)
-    print("raw_html=>", status_code, compress_html(raw_html, only_text=True))
-    print("\n")
+    raw_html = compress_html(raw_html, only_text=True)
+    # print("raw_html=>", status_code, compress_html(raw_html))
+    print("raw_html=>", status_code, raw_html)
+    page_html = dp_html(target_url)
+    page_html = compress_html(page_html, only_text=True)
+    # print("render_text=>", compress_html(page_html))
+    print("render_text_head=>", page_html)
+
+    page_html = dp_headless_html(target_url)
+    page_html = compress_html(page_html, only_text=True)
+    # print("render_text=>", compress_html(page_html))
+    print("render_text=>", page_html)
+
+    # print("\n")
+    # raw_html, status_code = requests_html(target_url)
+    # raw_html = compress_html(raw_html, only_text=True)
+    # # print("raw_html=>", status_code, compress_html(raw_html))
+    # print("raw_html=>", status_code, raw_html)
+    # print("\n")
 
 
 if __name__ == '__main__':
@@ -85,8 +139,25 @@ if __name__ == '__main__':
     # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
     # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
     # url = "https://www.nmpa.gov.cn/yaowen/ypjgyw/index.html"
-    url = "http://www.customs.gov.cn/customs/xwfb34/302425/index.html"
-    # url = "https://www.acftu.org/xwdt/ghyw/"
-
-    for i in range(20):
-        test(url)
+    # url = "http://www.customs.gov.cn/customs/xwfb34/302425/index.html"
+    # url = "https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html"
+    # url = "https://www.gjxfj.gov.cn/gjxfj/news/ttxw.htm"
+    # url = "https://www.gsgh.org.cn/#/moreNews_?position=%E7%9C%81%E6%80%BB%E6%96%B0%E9%97%BB&categoryId=502"
+    url = "https://www.acftu.org/xwdt/ghyw/"
+    # url = "https://www.nxzgh.org.cn/#/newsCenter/index2/2"  # 移动waf，会检测浏览器无头
+    # url = "https://www.chengdu.gov.cn/cdsrmzf/zfxx/cjj.shtml" # 超严格瑞数6，怀疑他会检测端口，我使用9222端口无论如何都获取不到结果
+    # url = "https://www.jsgh.org/col/col3577/index.html?uid=18462&pageNum=1"
+    # render_html = dp_headless_html(url)
+    # print("\n")
+    # print("render_html=>", render_html)
+    # for i in range(20):
+    test(url)
+    # todo: 大致盘一下各种判定的逻辑【以下的所有压缩比之间的差距均取“绝对值”】
+    #  1. 如果requests、无头、有头获取到的压缩比之间从差距都在15%以内，则认定该页面是静态页面，此时优先使用requests请求
+    #  2. 如果requests的status_code为特定的412，或者521，则判定是瑞数和jsl。[此时还有一个特点：requests的压缩比会与其他两种方式获取到的压缩比差距非常大(一两千的那种)]
+    #  3. 如果requests、无头、有头获取到的压缩比之间差距都在40%以上，则判定该页面只可以用有头采集
+    #  4. 如果无头和有头获取到的压缩比之间差距小于15%，但是requests和无头的差距大于40%，则认定该页面可以使用无头浏览器采集
+    #  5. 如果requests和有头获取到的压缩比之间差距小于15%，但是无头和有头的差距大于40%，则认定该页面优先使用有头浏览器采集
+    #  【此时可能是：1.使用了别的检测无头的waf。2.网站使用瑞数，但是这次请求没有拦截requests（不知道是不是瑞数那边故意设置的），
+    #   此时如果想进一步判定是否是瑞数，可以使用有头浏览器取一下cookies，如果cookies里面存在瑞数的cookie，那么就可以断定是瑞数】
+    #  6.

jarvis_brain-0.1.2.dist-info/RECORD

@@ -1,11 +0,0 @@
-mcp_tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-mcp_tools/dp_tools.py,sha256=xtpJoRmpqZQRjfVUEIjo3KAd5_SqK01J1VAt8U79xbA,8448
-mcp_tools/main.py,sha256=lxH8PafR4KVM0-OURgHD0jNaC68kLmPYaNJ1wSKfNdY,654
-mcp_tools/requests_tools.py,sha256=K1eHbvhzWhLzn9AZhjDGTmDu_yipvkEM9IOhveDMsPM,1063
-tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tools/simhash_tools.py,sha256=eRxUqcWJiNMNZPpVRBk1kI7OYSK_vThoVg3jctFfmhY,7368
-tools/tools.py,sha256=lPlXdVA4Dx3irP6alMOgmE-HxxuN_y4XQ3Q-fZyV-OU,3482
-jarvis_brain-0.1.2.dist-info/METADATA,sha256=Otgmq6QJe4XoWiIhC1i6jBpQsgWhraoPpi8v8Su7SYw,250
-jarvis_brain-0.1.2.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87
-jarvis_brain-0.1.2.dist-info/entry_points.txt,sha256=YFQT4xpkUqt5dM5wlKPQQOqcjMuFrT9iuRAzIpAyH7U,51
-jarvis_brain-0.1.2.dist-info/RECORD,,