beswarm 0.2.24__py3-none-any.whl → 0.2.25__py3-none-any.whl

beswarm/tools/search_web.py

@@ -4,6 +4,7 @@ import json
 import httpx
 from urllib.parse import quote_plus
 import threading
+import time
 
 from ..aient.src.aient.plugins import register_tool, get_url_content # Assuming a similar plugin structure
 
@@ -101,6 +102,17 @@ async def search_web(query: str):
                 except ValueError:
                     pass
 
+                # 2.5. 解码 Unicode 转义序列 (例如 \u003d -> =)
+                try:
+                    def replace_unicode(match):
+                        return chr(int(match.group(1), 16))
+                    # 只查找和替换 \uXXXX 格式的序列
+                    url_str = re.sub(r'\\u([0-9a-fA-F]{4})', replace_unicode, url_str)
+                except (ValueError, TypeError):
+                    # 如果转换失败（例如，格式错误的序列），则忽略
+                    print(f"Error decoding Unicode escape sequence in URL: {url_str}")
+                    pass
+
                 # 3. 解码 HTML 实体 (例如 & -> &)
                 url_str = html.unescape(url_str)
 
@@ -142,20 +154,35 @@ async def search_web(query: str):
             excluded_domains = [
                 "www.w3.org",
                 "www.google.com",
-                "ssl.gstatic.com",
                 "translate.google.com",
+                "id.google.com",
+                "lens.google.com",
+                "ssl.gstatic.com",
                 "www.googleadservices.com",
                 "gstatic.com",
-                "lens.google.com",
                 "schema.org",
-                "id.google.com",
                 "maps.google.com",
                 "clients6.google.com",
                 "ogs.google.com",
                 "policies.google.com",
                 "support.google.com",
                 "tpc.googlesyndication.com",
-                "adssettings.google.com"
+                "adssettings.google.com",
+            ]
+
+            full_excluded_urls = [
+                "https://google.com",
+                "https://patents.google.com",
+                "https://patentpc.com",
+                "https://www.mdpi.com",
+                "https://trackobit.com",
+                "https://www.researchgate.net",
+                "https://www.sciencedirect.com",
+                "https://rosap.ntl.bts.gov",
+                "https://portal.unifiedpatents.com",
+                "https://ieeexplore.ieee.org",
+                "https://files-backend.assets.thrillshare.com",
+                "https://patentimages.storage.googleapis.com",
             ]
 
             final_urls_before_dedup = []
@@ -171,7 +198,7 @@ async def search_web(query: str):
                     if normalized_url and not normalized_url.startswith(('http://', 'https://')):
                         normalized_url = 'https://' + normalized_url
 
-                    if normalized_url:
+                    if normalized_url and normalized_url not in full_excluded_urls:
                          final_urls_before_dedup.append(normalized_url)
 
             # 10. 去重
@@ -188,16 +215,16 @@ async def search_web(query: str):
     if results and isinstance(results, list) and len(results) > 0:
         # print(f"Fetching content for {len(results)} URLs...")
 
-        threads = []
+        threads_with_links = []
         for i, link in enumerate(results):
             print(f"Processing URL {i + 1}/{len(results)}: {link}")
             # Assuming get_url_content is synchronous and returns a string or None
             # content_text = get_url_content(link)
             url_search_thread = ThreadWithReturnValue(target=get_url_content, args=(link,))
             url_search_thread.start()
-            threads.append(url_search_thread)
+            threads_with_links.append((url_search_thread, link))
 
-        for thread in threads:
+        for thread, link in threads_with_links:
             content_text = thread.join()
             # content_text = thread.get_result()
             if content_text and len(content_text.split("\n\n")) > 10: # Ensure content_text is not None or empty before adding
@@ -229,6 +256,7 @@ async def search_web(query: str):
     to_keep_flags = [True] * n  # Flags to mark which items to keep
 
     # print("Starting similarity comparison...")
+    # start_time = time.time()
     for i in range(n):
         if not to_keep_flags[i]:  # Skip if item i is already marked for discard
             continue
@@ -246,14 +274,16 @@ async def search_web(query: str):
                 content_j = str(content_j) # Fallback
 
             similarity = calculate_similarity(content_i, content_j)
+            # print(f"Similarity between {web_contents_raw[i]['url']} and {web_contents_raw[j]['url']}: {similarity:.4f}")
 
-            if similarity > 0.9:
+            if similarity > 0.5:
                 # print(f"Similarity > 0.9 ({similarity:.4f}) between content from '{web_contents_raw[i]['url']}' and '{web_contents_raw[j]['url']}'. Discarding the latter.")
                 to_keep_flags[j] = False  # Discard the second item (item j)
 
     final_web_content = [web_contents_raw[i] for i in range(n) if to_keep_flags[i]]
     # print(f"Number of items after filtering: {len(final_web_content)}")
-
+    # end_time = time.time()
+    # print(f"Time taken: {end_time - start_time:.2f} seconds")
     # output_filename = "web_content_filtered.json"
     # with open(output_filename, "w", encoding="utf-8") as f:
     #     json.dump(final_web_content, f, indent=2, ensure_ascii=False)
@@ -270,16 +300,56 @@ import difflib
 
 
 def calculate_similarity(string1: str, string2: str) -> float:
-    """Calculates the similarity ratio between two strings.
-
-    Args:
-        string1: The first string.
-        string2: The second string.
-
-    Returns:
-        A float between 0 and 1, where 1 means the strings are identical
-        and 0 means they are completely different.
     """
+    根据您的最终反馈，整合了多级筛选策略来优化性能，且所有修改均在函数内部。
+
+    优化思路:
+    1.  长度筛选: 使用“min/max比例法”进行快速检查。如果difflib相似度的
+        数学上限已经低于主循环中使用的阈值(0.5)，则直接退出。
+    2.  分块筛选: 采纳您提出的分块思想。我们将较短的字符串切分为20个块，
+        并快速计算有多少块也出现在另一个字符串中。这是一个成本远低于difflib的内容预筛选。
+        - 如果重合度很高 (>80%)，可以提前判断为相似。
+        - 如果重合度很低 (<20%)，可以提前判断为不相似。
+    3.  最终精确计算: 只有当相似度处于“中间地带”，前两级筛选无法确定时，
+        我们才动用最精确但最耗时的difflib进行最终裁决。
+    """
+    len1, len2 = len(string1), len(string2)
+
+    # 第一级筛选: 长度检查 (非常廉价)
+    # 2.0 * min(len1, len2) / (len1 + len2) 是 difflib.ratio() 的数学上限。
+    # 这里的阈值0.5必须与主循环中的 `if similarity > 0.5:` 保持一致。
+    # print(len1, len2, (2.0 * min(len1, len2) / (len1 + len2)))
+    if not len1 or not len2 or (2.0 * min(len1, len2) / (len1 + len2)) < 0.5:
+        return 0.0
+
+    # 对于短字符串，分块没有意义，直接比较
+    if len1 < 40 or len2 < 40:
+        return difflib.SequenceMatcher(None, string1, string2).ratio()
+
+    # 第二级筛选: 分块检查 (中等成本)
+    shorter_str, longer_str = (string1, string2) if len1 < len2 else (string2, string1)
+
+    num_chunks = 1000
+    chunk_size = len(shorter_str) // num_chunks
+
+    # 因为上面已经有len < 40的检查，这里的chunk_size不可能为0，所以之前的if chunk_size == 0是冗余的。
+    matching_chunks = 0
+    for i in range(num_chunks):
+        start = i * chunk_size
+        chunk = shorter_str[start:start+chunk_size]
+        if chunk in longer_str:
+            matching_chunks += 1
+
+    match_ratio = matching_chunks / num_chunks
+    # print(matching_chunks, match_ratio)
+
+    # 根据分块匹配率进行判断，这些阈值是基于经验的启发式规则。
+    if match_ratio > 0.8:  # 超过80%的块匹配，几乎可以肯定是高度相似
+        return match_ratio # 返回一个确保能通过主循环判断的高值
+    if match_ratio < 0.2: # 少于20%的块匹配，几乎不可能相似
+        return match_ratio
+
+    # 第三级：最终精确计算 (高成本)
     return difflib.SequenceMatcher(None, string1, string2).ratio()
 
 if __name__ == '__main__':
@@ -289,7 +359,9 @@ if __name__ == '__main__':
     async def main():
         # 示例用法
         # search_query = "美国"
-        search_query = "machine learning models for higher heating value prediction using proximate vs ultimate analysis"
+        # search_query = "machine learning models for higher heating value prediction using proximate vs ultimate analysis"
+        # search_query = "patent driver cognitive load monitoring micro-expression thermal imaging fusion"
+        search_query = "patent predictive driver fatigue warning V2X data fusion driving behavior sequence"
         print(f"Performing web search for: '{search_query}'")
         results = await search_web(search_query)  # results is a list of URLs
 

{beswarm-0.2.24.dist-info → beswarm-0.2.25.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: beswarm
-Version: 0.2.24
+Version: 0.2.25
 Summary: MAS
 Requires-Python: >=3.11
 Description-Content-Type: text/markdown

{beswarm-0.2.24.dist-info → beswarm-0.2.25.dist-info}/RECORD

@@ -133,10 +133,10 @@ beswarm/tools/repomap.py,sha256=YsTPq5MXfn_Ds5begcvHDnY_Xp2d4jH-xmWqNMHnNHY,4523
 beswarm/tools/request_input.py,sha256=gXNAJPOJektMqxJVyzNTFOeMQ7xUkO-wWMYH-r2Rdwk,942
 beswarm/tools/screenshot.py,sha256=u6t8FCgW5YHJ_Oc4coo8e0F3wTusWE_-H8dFh1rBq9Q,1011
 beswarm/tools/search_arxiv.py,sha256=caVIUOzMhFu-r_gVgJZrH2EO9xI5iV_qLAg0b3Ie9Xg,8095
-beswarm/tools/search_web.py,sha256=tLdw63doMTorrCG3ZoQkKvQPYBdx-m-SJskAXxfdim8,11958
+beswarm/tools/search_web.py,sha256=eEE_aRcocttAwWfkcQdElI_BZw73xiRIEfbHDWAoQqU,15996
 beswarm/tools/taskmanager.py,sha256=oB_768qy6Lb58JNIcSLVgbPrgNB3duIq9DawbVHRbrg,6270
 beswarm/tools/worker.py,sha256=Vwn1XuTZ2dIStd5dQ6DhJ4f7LmwDc-Sx9PwLG0Xw-MQ,24062
-beswarm-0.2.24.dist-info/METADATA,sha256=PbLmlEKhy1mc1phhuG61O-XtynVlNWpG0y0NKB9Szho,3847
-beswarm-0.2.24.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-beswarm-0.2.24.dist-info/top_level.txt,sha256=pJw4O87wvt5882smuSO6DfByJz7FJ8SxxT8h9fHCmpo,8
-beswarm-0.2.24.dist-info/RECORD,,
+beswarm-0.2.25.dist-info/METADATA,sha256=Wwq8gqLH0xv1KMUkNsbfZEw6XRGKNORHzNTMXyPLjxs,3847
+beswarm-0.2.25.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+beswarm-0.2.25.dist-info/top_level.txt,sha256=pJw4O87wvt5882smuSO6DfByJz7FJ8SxxT8h9fHCmpo,8
+beswarm-0.2.25.dist-info/RECORD,,