PyPI - Jarvis-Brain - Versions diffs - 0.1.2__tar.gz → 0.1.3__tar.gz - Mend

Jarvis-Brain 0.1.2tar.gz → 0.1.3tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (20) hide show

{jarvis_brain-0.1.2 → jarvis_brain-0.1.3}/.idea/workspace.xml RENAMED Viewed

@@ -13,8 +13,8 @@
   <component name="FileTemplateManagerImpl">
     <option name="RECENT_TEMPLATES">
       <list>
-        <option value="HTML File" />
         <option value="Python Script" />
+        <option value="HTML File" />
       </list>
     </option>
   </component>
@@ -26,24 +26,25 @@
     <option name="hideEmptyMiddlePackages" value="true" />
     <option name="showLibraryContents" value="true" />
   </component>
-  <component name="PropertiesComponent">{
-  &quot;keyToString&quot;: {
-    &quot;DefaultHtmlFileTemplate&quot;: &quot;HTML File&quot;,
-    &quot;ModuleVcsDetector.initialDetectionPerformed&quot;: &quot;true&quot;,
-    &quot;Python.1.executor&quot;: &quot;Run&quot;,
-    &quot;Python.2.executor&quot;: &quot;Run&quot;,
-    &quot;Python.dp_tools.executor&quot;: &quot;Run&quot;,
-    &quot;Python.main.executor&quot;: &quot;Run&quot;,
-    &quot;Python.requests_tools.executor&quot;: &quot;Run&quot;,
-    &quot;Python.simhash.executor&quot;: &quot;Run&quot;,
-    &quot;Python.simhash_tools.executor&quot;: &quot;Run&quot;,
-    &quot;Python.tools.executor&quot;: &quot;Run&quot;,
-    &quot;RunOnceActivity.ShowReadmeOnStart&quot;: &quot;true&quot;,
-    &quot;RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252&quot;: &quot;true&quot;,
-    &quot;last_opened_file_path&quot;: &quot;/Users/user/PycharmProjects/JARVIS&quot;,
-    &quot;settings.editor.selected.configurable&quot;: &quot;com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable&quot;
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "DefaultHtmlFileTemplate": "HTML File",
+    "ModuleVcsDetector.initialDetectionPerformed": "true",
+    "Python.1.executor": "Run",
+    "Python.2.executor": "Run",
+    "Python.3.executor": "Run",
+    "Python.dp_tools.executor": "Run",
+    "Python.main.executor": "Run",
+    "Python.requests_tools.executor": "Run",
+    "Python.simhash.executor": "Run",
+    "Python.simhash_tools.executor": "Run",
+    "Python.tools.executor": "Run",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "RunOnceActivity.TerminalTabsStorage.copyFrom.TerminalArrangementManager.252": "true",
+    "last_opened_file_path": "/Users/user/PycharmProjects/JARVIS",
+    "settings.editor.selected.configurable": "com.jetbrains.python.configuration.PyActiveSdkModuleConfigurable"
   }
-}</component>
+}]]></component>
   <component name="RecentsManager">
     <key name="CopyFile.RECENT_KEYS">
       <recent name="$PROJECT_DIR$" />
@@ -99,7 +100,7 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
-    <configuration name="dp_tools" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
+    <configuration name="3" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
       <module name="JARVIS" />
       <option name="ENV_FILES" value="" />
       <option name="INTERPRETER_OPTIONS" value="" />
@@ -108,11 +109,11 @@
         <env name="PYTHONUNBUFFERED" value="1" />
       </envs>
       <option name="SDK_HOME" value="" />
-      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/task_router" />
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/src/mcp_tools/dp_tools.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/task_router/3.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
@@ -121,7 +122,7 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
-    <configuration name="requests_tools" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+    <configuration name="dp_tools" type="PythonConfigurationType" factoryName="Python" nameIsGenerated="true">
       <module name="JARVIS" />
       <option name="ENV_FILES" value="" />
       <option name="INTERPRETER_OPTIONS" value="" />
@@ -130,11 +131,11 @@
         <env name="PYTHONUNBUFFERED" value="1" />
       </envs>
       <option name="SDK_HOME" value="" />
-      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/mcp_tools" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$" />
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/mcp_tools/requests_tools.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/src/mcp_tools/dp_tools.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
@@ -143,7 +144,7 @@
       <option name="INPUT_FILE" value="" />
       <method v="2" />
     </configuration>
-    <configuration name="simhash_tools" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
+    <configuration name="requests_tools" type="PythonConfigurationType" factoryName="Python" temporary="true" nameIsGenerated="true">
       <module name="JARVIS" />
       <option name="ENV_FILES" value="" />
       <option name="INTERPRETER_OPTIONS" value="" />
@@ -152,11 +153,11 @@
         <env name="PYTHONUNBUFFERED" value="1" />
       </envs>
       <option name="SDK_HOME" value="" />
-      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/tools" />
+      <option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/mcp_tools" />
       <option name="IS_MODULE_SDK" value="true" />
       <option name="ADD_CONTENT_ROOTS" value="true" />
       <option name="ADD_SOURCE_ROOTS" value="true" />
-      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/tools/simhash_tools.py" />
+      <option name="SCRIPT_NAME" value="$PROJECT_DIR$/mcp_tools/requests_tools.py" />
       <option name="PARAMETERS" value="" />
       <option name="SHOW_COMMAND_LINE" value="false" />
       <option name="EMULATE_TERMINAL" value="false" />
@@ -191,9 +192,9 @@
       <list>
         <item itemvalue="Python.1" />
         <item itemvalue="Python.tools" />
+        <item itemvalue="Python.3" />
         <item itemvalue="Python.2" />
         <item itemvalue="Python.requests_tools" />
-        <item itemvalue="Python.simhash_tools" />
       </list>
     </recent_temporary>
   </component>
@@ -214,4 +215,15 @@
     </task>
     <servers />
   </component>
+  <component name="XDebuggerManager">
+    <breakpoint-manager>
+      <breakpoints>
+        <line-breakpoint enabled="true" suspend="THREAD" type="python-line">
+          <url>file://$PROJECT_DIR$/tools/tools.py</url>
+          <line>16</line>
+          <option name="timeStamp" value="1" />
+        </line-breakpoint>
+      </breakpoints>
+    </breakpoint-manager>
+  </component>
 </project>

{jarvis_brain-0.1.2 → jarvis_brain-0.1.3}/PKG-INFO RENAMED Viewed

@@ -1,9 +1,10 @@
 Metadata-Version: 2.4
 Name: Jarvis_Brain
-Version: 0.1.2
+Version: 0.1.3
 Summary: Jarvis brain
 Requires-Python: >=3.10
 Requires-Dist: beautifulsoup4
+Requires-Dist: curl-cffi
 Requires-Dist: drissionpage
 Requires-Dist: fastmcp
 Requires-Dist: htmlmin

{jarvis_brain-0.1.2 → jarvis_brain-0.1.3}/mcp_tools/dp_tools.py RENAMED Viewed

@@ -1,9 +1,9 @@
 import json
 import os
 import random
-from pyexpat.errors import messages
 from typing import Any
+from urllib.parse import urlparse
 from DrissionPage import ChromiumPage, ChromiumOptions
 from DrissionPage._elements.none_element import NoneElement
 from fastmcp import FastMCP
@@ -209,3 +209,23 @@ def register_assert_Static_Web(mcp: FastMCP):
                 }, ensure_ascii=False)
             }]
         }
+def register_get_cookies(mcp: FastMCP):
+    @mcp.tool(name="get_cookies")
+    def get_cookies(browser_port: int, tab_id: str) -> dict[str, Any]:
+        _browser = get_page(browser_port)
+        tab = _browser.get_tab(tab_id)
+        target_url = tab.url
+        parser = urlparse(target_url)
+        domain = parser.netloc
+        cookies = tab.cookies()
+        # tab.cookies
+        return {
+            "content": [{
+                "type": "text",
+                "text": json.dumps({
+                }, ensure_ascii=False)
+            }]
+        }

{jarvis_brain-0.1.2 → jarvis_brain-0.1.3}/mcp_tools/requests_tools.py RENAMED Viewed

@@ -1,19 +1,26 @@
 import json
 from fastmcp import FastMCP
-from tools.tools import requests_html
-from DrissionPage import ChromiumPage, ChromiumOptions
+from tools.tools import requests_html, dp_headless_html
 def register_assert_waf(mcp: FastMCP):
     @mcp.tool(name="assert_waf", description="判断传入的url对应的网页是否存在瑞数、jsl等风控防火墙")
     def assert_waf(url: str):
+        # 先试用requests判断瑞数和jsl
         text, code = requests_html(url)
         waf_text_type = {
             521: "jsl",
-            412: "瑞数"
+            412: "瑞数",
+            "触发WAF": "其他waf"
         }
         has_waf = code in waf_text_type.keys()
+        if not has_waf:
+            # 不是瑞数和jsl，使用无头浏览器判断是否为其他waf
+            headless_html = dp_headless_html(url)
+            if "触发WAF" in headless_html:
+                has_waf = True
+                code = "触发WAF"
         if not has_waf:
             waf_type = "不存在waf"
         else:
@@ -31,6 +38,3 @@ def register_assert_waf(mcp: FastMCP):
                 )
             }]
         }

{jarvis_brain-0.1.2 → jarvis_brain-0.1.3}/pyproject.toml RENAMED Viewed

@@ -1,6 +1,6 @@
 [project]
 name = "Jarvis_Brain"  # 别人下载时用的名字，必须在 PyPI 上唯一
-version = "0.1.2"
+version = "0.1.3"
 description = "Jarvis brain"
 dependencies = [
     "fastmcp",
@@ -9,7 +9,7 @@ dependencies = [
     "jieba",
     "simhash",
     "beautifulsoup4",
-    # 在这里列出 dp_tools.py 需要的其他库，例如 "pandas", "requests"
+    "curl_cffi"
 ]
 requires-python = ">=3.10"

{jarvis_brain-0.1.2 → jarvis_brain-0.1.3}/task_router/1.py RENAMED Viewed

@@ -51,41 +51,64 @@ options = ClaudeAgentOptions(
     permission_mode='bypassPermissions',
     cwd=base_cwd,
     mcp_servers={
-        "team-node-dp": {
-            "command": "uv",
-            "args": [
-                "run",
-                "--directory",
-                "/Users/user/PycharmProjects/JARVIS/mcp_tools",
-                "main.py"
-            ],
+        "jarvis": {
+            "command": "uvx",
+            "args": ["--from", "jarvis_brain", "jarvis-mcp"],
             "env": {
                 "MCP_MODULES": "TeamNode-Dp"
             }
         },
-        "jarvis-node": {
-            "command": "uv",
-            "args": [
-                "run",
-                "--directory",
-                "/Users/user/PycharmProjects/JARVIS/mcp_tools",
-                "main.py"
-            ],
-            "env": {
-                "MCP_MODULES": "JarvisNode"
-            }
-        },
+        # "team-node-dp": {
+        #     "command": "uv",
+        #     "args": [
+        #         "run",
+        #         "--directory",
+        #         "/Users/user/PycharmProjects/JARVIS/mcp_tools",
+        #         "main.py"
+        #     ],
+        #     "env": {
+        #         "MCP_MODULES": "TeamNode-Dp"
+        #     }
+        # },
+        # "jarvis-node": {
+        #     "command": "uv",
+        #     "args": [
+        #         "run",
+        #         "--directory",
+        #         "/Users/user/PycharmProjects/JARVIS/mcp_tools",
+        #         "main.py"
+        #     ],
+        #     "env": {
+        #         "MCP_MODULES": "JarvisNode"
+        #     }
+        # },
     },
     # setting_sources=["project"],
+    # allowed_tools=[
+    #     "mcp__team-node-dp__get_html",
+    #     "mcp__team-node-dp__visit_url",
+    #     "mcp__team-node-dp__get_new_tab",
+    #     "mcp__team-node-dp__switch_tab",
+    #     "mcp__team-node-dp__close_tab",
+    #     "mcp__team-node-dp__check_selector",
+    #     "mcp__jarvis-node__assert_Static_Web",  # 判断当前页面是否为静态页面
+    #     "mcp__jarvis-node__assert_waf",  # 判断传入的url是否使用了瑞数，jsl等防火墙
+    #     'Read',
+    #     'Write',
+    #     'Edit',
+    #     'MultiEdit',
+    #     'Grep',
+    #     'Glob',
+    #     'TodoWrite'
+    # ]
     allowed_tools=[
-        "mcp__team-node-dp__get_html",
-        "mcp__team-node-dp__visit_url",
-        "mcp__team-node-dp__get_new_tab",
-        "mcp__team-node-dp__switch_tab",
-        "mcp__team-node-dp__close_tab",
-        "mcp__team-node-dp__check_selector",
-        "mcp__jarvis-node__assert_Static_Web",  # 判断当前页面是否为静态页面
-        "mcp__jarvis-node__assert_waf",  # 判断传入的url是否使用了瑞数，jsl等防火墙
+        "mcp__jarvis__get_html",
+        "mcp__jarvis__visit_url",
+        "mcp__jarvis__get_new_tab",
+        "mcp__jarvis__switch_tab",
+        "mcp__jarvis__close_tab",
+        "mcp__jarvis__check_selector",
+        "mcp__jarvis__check_selector",
         'Read',
         'Write',
         'Edit',
@@ -136,7 +159,7 @@ async def main():
         tasks.append((run, (), {  # 关键字参数字典
             "url": value,
         }))
-    manager = SimpleTaskManager(max_concurrent=1)
+    manager = SimpleTaskManager(max_concurrent=5)
     results = await manager.process_all(tasks)
     # 输出结果
     print("\n任务结果摘要:")

jarvis_brain-0.1.3/tools/tools.py ADDED Viewed

@@ -0,0 +1,163 @@
+import time
+import htmlmin
+from curl_cffi import requests
+from lxml import html, etree
+from bs4 import BeautifulSoup
+from DrissionPage import ChromiumPage, ChromiumOptions
+# 传入requests的set-cookie的str，返回一个cookie dict
+def cookie_str2dict(cookie_str: str):
+    if cookie_str == "":
+        return {}
+    cookie_dict = {}
+    cookie_list = [cookie.split(";")[0] for cookie in cookie_str.split("HttpOnly,")]
+    for cookie in cookie_list:
+        key, value = cookie.split("=")
+        cookie_dict[key] = value
+    return cookie_dict
+# 使用requests获取html，用于测试是否使用了瑞数和jsl
+def requests_html(url):
+    headers = {
+        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
+    }
+    response = requests.get(url, headers=headers, verify=False)
+    # print("response headers=> ", type(response.headers.get("Set-Cookie")),
+    #       cookie_str2dict(response.headers.get("Set-Cookie", "")))
+    response.encoding = "utf-8"
+    return response.text, response.status_code
+# 使用dp无头模式获取html，用于测试是否使用了其他waf，如移动waf
+def dp_headless_html(url):
+    opt = ChromiumOptions().headless(True)
+    opt.set_argument('--no-sandbox')
+    opt.set_local_port(9976)
+    page = ChromiumPage(opt)
+    # page = ChromiumPage()
+    tab = page.latest_tab
+    # tab.set.load_mode.normal()
+    tab.get(url)
+    # tab.wait.eles_loaded()
+    time.sleep(10)
+    # tab.wait.load_start()
+    page_html = tab.html
+    # print(page_html)
+    # print("dp_cookies=>", tab.cookies())
+    page.quit()
+    return page_html
+# 使用dp无头模式获取html，用于测试是否使用了其他waf，如移动waf
+def dp_html(url):
+    opt = ChromiumOptions()
+    opt.set_local_port(9223)
+    # opt.set_argument('--no-sandbox')
+    page = ChromiumPage(opt)
+    # page = ChromiumPage()
+    tab = page.latest_tab
+    # tab.set.load_mode.normal()
+    tab.get(url)
+    # tab.wait.eles_loaded()
+    # time.sleep(10)
+    tab.wait.doc_loaded()
+    page_html = tab.html
+    # print(page_html)
+    # print("dp_cookies=>", tab.cookies())
+    page.quit()
+    return page_html
+# 压缩html
+def compress_html(content, only_text=False):
+    doc = html.fromstring(content)
+    # 删除 style 和 script 标签
+    for element in doc.xpath('//style | //script'):
+        element.getparent().remove(element)
+    # 删除 link 标签
+    for link in doc.xpath('//link[@rel="stylesheet"]'):
+        link.getparent().remove(link)
+    # 删除 meta 标签（新增功能）
+    for meta in doc.xpath('//meta'):
+        meta.getparent().remove(meta)
+    # 删除 style 属性
+    for element in doc.xpath('//*[@style]'):
+        element.attrib.pop('style')
+    # 删除所有 on* 事件属性
+    for element in doc.xpath('//*'):
+        for attr in list(element.attrib.keys()):
+            if attr.startswith('on'):
+                element.attrib.pop(attr)
+    result = etree.tostring(doc, encoding='unicode')
+    result = htmlmin.minify(result)
+    print(f"html压缩比=> {len(content) / len(result) * 100:.2f}%")
+    if not only_text:
+        return result
+    soup = BeautifulSoup(result, 'html.parser')
+    result = soup.get_text(strip=True)
+    return result
+def test(target_url):
+    raw_html, status_code = requests_html(target_url)
+    raw_html = compress_html(raw_html, only_text=True)
+    # print("raw_html=>", status_code, compress_html(raw_html))
+    print("raw_html=>", status_code, raw_html)
+    page_html = dp_html(target_url)
+    page_html = compress_html(page_html, only_text=True)
+    # print("render_text=>", compress_html(page_html))
+    print("render_text_head=>", page_html)
+    page_html = dp_headless_html(target_url)
+    page_html = compress_html(page_html, only_text=True)
+    # print("render_text=>", compress_html(page_html))
+    print("render_text=>", page_html)
+    # print("\n")
+    # raw_html, status_code = requests_html(target_url)
+    # raw_html = compress_html(raw_html, only_text=True)
+    # # print("raw_html=>", status_code, compress_html(raw_html))
+    # print("raw_html=>", status_code, raw_html)
+    # print("\n")
+if __name__ == '__main__':
+    # raw_html, status_code = requests_html("https://www.nxzgh.org.cn/#/newsCenter/index2/2")
+    # raw_html, status_code = requests_html("http://www.ncha.gov.cn/col/col722/index.html")
+    # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
+    # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
+    # url = "https://www.nmpa.gov.cn/yaowen/ypjgyw/index.html"
+    # url = "http://www.customs.gov.cn/customs/xwfb34/302425/index.html"
+    # url = "https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html"
+    # url = "https://www.gjxfj.gov.cn/gjxfj/news/ttxw.htm"
+    # url = "https://www.gsgh.org.cn/#/moreNews_?position=%E7%9C%81%E6%80%BB%E6%96%B0%E9%97%BB&categoryId=502"
+    url = "https://www.acftu.org/xwdt/ghyw/"
+    # url = "https://www.nxzgh.org.cn/#/newsCenter/index2/2"  # 移动waf，会检测浏览器无头
+    # url = "https://www.chengdu.gov.cn/cdsrmzf/zfxx/cjj.shtml" # 超严格瑞数6，怀疑他会检测端口，我使用9222端口无论如何都获取不到结果
+    # url = "https://www.jsgh.org/col/col3577/index.html?uid=18462&pageNum=1"
+    # render_html = dp_headless_html(url)
+    # print("\n")
+    # print("render_html=>", render_html)
+    # for i in range(20):
+    test(url)
+    # todo: 大致盘一下各种判定的逻辑【以下的所有压缩比之间的差距均取“绝对值”】
+    #  1. 如果requests、无头、有头获取到的压缩比之间从差距都在15%以内，则认定该页面是静态页面，此时优先使用requests请求
+    #  2. 如果requests的status_code为特定的412，或者521，则判定是瑞数和jsl。[此时还有一个特点：requests的压缩比会与其他两种方式获取到的压缩比差距非常大(一两千的那种)]
+    #  3. 如果requests、无头、有头获取到的压缩比之间差距都在40%以上，则判定该页面只可以用有头采集
+    #  4. 如果无头和有头获取到的压缩比之间差距小于15%，但是requests和无头的差距大于40%，则认定该页面可以使用无头浏览器采集
+    #  5. 如果requests和有头获取到的压缩比之间差距小于15%，但是无头和有头的差距大于40%，则认定该页面优先使用有头浏览器采集
+    #  【此时可能是：1.使用了别的检测无头的waf。2.网站使用瑞数，但是这次请求没有拦截requests（不知道是不是瑞数那边故意设置的），
+    #   此时如果想进一步判定是否是瑞数，可以使用有头浏览器取一下cookies，如果cookies里面存在瑞数的cookie，那么就可以断定是瑞数】
+    #  6.

{jarvis_brain-0.1.2 → jarvis_brain-0.1.3}/uv.lock RENAMED Viewed

@@ -585,7 +585,7 @@ wheels = [
 [[package]]
 name = "jarvis-brain"
-version = "0.1.1"
+version = "0.1.2"
 source = { editable = "." }
 dependencies = [
     { name = "beautifulsoup4" },

jarvis_brain-0.1.2/task_router/2.py DELETED Viewed

@@ -1,14 +0,0 @@
-from tools.simhash_tools import HTMLSimHashComparator
-text1 = "四川省总工会官网欢迎访问四川省总工会领导信箱微 信微 博首    页工会概况新闻中心部门动态信息公开媒体矩阵工作平台首页>新闻中心>工会新闻 >主办单位：四川省总工会 Copyright 2021 www.scgh.org All Rights Reserved.电话:(028)86255956  (028)86122339邮编:610000川公网安备 51010502010464号    技术单位 :四川银利华应用科技有限责任公司建议浏览器：360、谷歌、火狐、IE10蜀ICP备06003706号-1总访问量：21392491 今年：14426832 本月：7210766 今日：6061"
-text2 = "四川省总工会官网欢迎访问四川省总工会领导信箱微 信微 博首    页工会概况新闻中心部门动态信息公开媒体矩阵工作平台首页>新闻中心>工会新闻 >粽叶裹童心 四川工会解锁“1+1>2“的快乐公式——我省各级工会组织开展2025年“六一”儿童节关爱活动[2025-06-01]四川省工会资产监督管理工作现场会在宜宾召开[2025-05-30]“六一”关爱来袭！全国爱心托育用人单位经验在这里开花[2025-05-29]“引智助企”示范活动暨“劳模工匠助企行”数智化应用专项行动在乐山举行[2025-05-29]四川省交通运输行业开展职工安全生产“大培训、大测试、大比武”[2025-05-28]省总女职委巾帼劳模工匠学习贯彻习近平总书记重要讲话精神座谈会召开[2025-05-28]四川省工会党支部书记培训班开班[2025-05-20]中国职工保险互助会来川开展座谈交流[2025-05-19]四川省劳模和先进工作者座谈会在蓉召开[2025-05-16]川渝滇黔浙五省市茶界高手首聚蒙顶山“论剑”[2025-05-15]共 5207 条上一页12345…521下一页10 条/页20 条/页30 条/页40 条/页50 条/页到第页确定主办单位：四川省总工会 Copyright 2021 www.scgh.org All Rights Reserved.电话:(028)86255956  (028)86122339邮编:610000川公网安备 51010502010464号    技术单位 :四川银利华应用科技有限责任公司建议浏览器：360、谷歌、火狐、IE10蜀ICP备06003706号-1总访问量：21392491 今年：14426832 本月：7210766 今日：6061"
-comparator = HTMLSimHashComparator(text1, text2)
-result = comparator.compare_simhash()
-print("SimHash比较结果:")
-print(f"页面1 SimHash: {result['simhash1']}")
-print(f"页面2 SimHash: {result['simhash2']}")
-print(f"汉明距离: {result['hamming_distance']}")
-print(f"相似度: {result['similarity_percentage']}")
-threshold_result = comparator.compare_with_threshold(threshold=0.7)
-print(f"\n基于阈值{threshold_result['threshold']}的判断:")
-print(f"是否相似: {threshold_result['is_similar']}")

jarvis_brain-0.1.2/tools/tools.py DELETED Viewed

@@ -1,92 +0,0 @@
-import htmlmin
-import requests
-from lxml import html, etree
-from bs4 import BeautifulSoup
-from DrissionPage import ChromiumPage, ChromiumOptions
-# 使用requests获取html，用于测试是否使用了瑞数和jsl
-def requests_html(url):
-    headers = {
-        # "sec-ch-ua": "\"Chromium\";v=\"142\", \"Google Chrome\";v=\"142\", \"Not_A Brand\";v=\"99\"",
-        # "sec-ch-ua-mobile": "?0",
-        # "Upgrade-Insecure-Requests": "1",
-        "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/142.0.0.0 Safari/537.36",
-        # "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
-        # "Sec-Fetch-Site": "none",
-        # "Sec-Fetch-Mode": "navigate",
-        # "Sec-Fetch-User": "?1",
-        # "Sec-Fetch-Dest": "document",
-        # "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8"
-    }
-    response = requests.get(url, headers=headers, verify=False)
-    print("response headers=> ", response.headers)
-    return response.text, response.status_code
-# 使用dp无头模式获取html，用于测试是否使用了其他waf，如移动waf
-def dp_headless_html(url):
-    opt = ChromiumOptions().headless(True)
-    opt.set_argument('--no-sandbox')
-    page = ChromiumPage(opt)
-    page.get(url)
-    page_html = page.html
-    page.quit()
-    return page_html
-# 压缩html
-def compress_html(content, only_text=False):
-    doc = html.fromstring(content)
-    # 删除 style 和 script 标签
-    for element in doc.xpath('//style | //script'):
-        element.getparent().remove(element)
-    # 删除 link 标签
-    for link in doc.xpath('//link[@rel="stylesheet"]'):
-        link.getparent().remove(link)
-    # 删除 meta 标签（新增功能）
-    for meta in doc.xpath('//meta'):
-        meta.getparent().remove(meta)
-    # 删除 style 属性
-    for element in doc.xpath('//*[@style]'):
-        element.attrib.pop('style')
-    # 删除所有 on* 事件属性
-    for element in doc.xpath('//*'):
-        for attr in list(element.attrib.keys()):
-            if attr.startswith('on'):
-                element.attrib.pop(attr)
-    result = etree.tostring(doc, encoding='unicode')
-    result = htmlmin.minify(result)
-    print(f"html压缩比=> {len(content) / len(result) * 100:.2f}%")
-    if not only_text:
-        return result
-    soup = BeautifulSoup(result, 'html.parser')
-    result = soup.get_text(strip=True)
-    return result
-def test(target_url):
-    page_html = dp_headless_html(target_url)
-    print("render_text=>", compress_html(page_html, only_text=True))
-    print("\n")
-    raw_html, status_code = requests_html(target_url)
-    print("raw_html=>", status_code, compress_html(raw_html, only_text=True))
-    print("\n")
-if __name__ == '__main__':
-    # raw_html, status_code = requests_html("https://www.nxzgh.org.cn/#/newsCenter/index2/2")
-    # raw_html, status_code = requests_html("http://www.ncha.gov.cn/col/col722/index.html")
-    # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
-    # raw_html, status_code = requests_html("https://scgh.org/page/news/tmore36403294e28a11ea8ba48cec4b967595.html")
-    # url = "https://www.nmpa.gov.cn/yaowen/ypjgyw/index.html"
-    url = "http://www.customs.gov.cn/customs/xwfb34/302425/index.html"
-    # url = "https://www.acftu.org/xwdt/ghyw/"
-    for i in range(20):
-        test(url)