re-common 10.0.2__tar.gz → 10.0.4__tar.gz

{re_common-10.0.2 → re_common-10.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.2
+Version: 10.0.4
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.2 → re_common-10.0.4}/re_common/v2/baselibrary/utils/string_clear.py

@@ -1,7 +1,7 @@
 import re
 import regex
 
-from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant
+from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html
 
 
 class StringClear(object):
@@ -88,20 +88,9 @@ class StringClear(object):
 
     def remove_html_tag(self):
         import html
-        from parsel import Selector
 
         self.obj_str = html.unescape(self.obj_str)
 
-        def clean_html(html):
-            sel = Selector(text=html)
-            # 移除脚本和样式
-            sel.xpath('//script').remove()
-            sel.xpath('//style').remove()
-
-            # 提取文本
-            text_nodes = sel.xpath('//text()').getall()
-            return ''.join(t.strip() for t in text_nodes if t.strip())
-
         self.obj_str = clean_html(self.obj_str)
 
         return self
@@ -116,7 +105,7 @@ def rel_clear(str_obj):
             .None_to_str()  # 空对象转str 防止空对象
             .to_str()  # 防止其他类型传入 比如 int double
             .qj_to_bj()  # 全角转半角
-            .remove_html_tag() # html标签清理
+            .remove_html_tag()  # html标签清理
             .replace_dash_with_space()  # 横线转空格 在 英文 title 中更有用
             .remove_special_chars()  # 移除特殊字符，仅保留字母、数字、空格和汉字 \w 已经包括所有 Unicode 字母 下划线 _ 会被保留
             .collapse_spaces()  # 移除多余空格,连续多个空格变一个

{re_common-10.0.2 → re_common-10.0.4}/re_common/v2/baselibrary/utils/stringutils.py

@@ -1,7 +1,9 @@
 import re
+import threading
 
 import regex
 import unicodedata
+from html.parser import HTMLParser
 
 
 def bj2qj(src):
@@ -25,6 +27,7 @@ def bj2qj(src):
 
     return ''.join(buf)
 
+
 def qj2bj(src):
     """
     全角转半角
@@ -62,6 +65,7 @@ def get_diacritic_variant(char1):
     # 判断基本字符是否相同
     return base_char1
 
+
 def get_alphabetic_ratio(text: str) -> float:
     # 返回字母型字符所占比例
     if not text:
@@ -93,4 +97,50 @@ def get_alphabetic_ratio(text: str) -> float:
     alphabetic_chars = re.findall(alphabetic_pattern, clean_text)
 
     # 返回字母型字符所占比例
-    return len(alphabetic_chars) / len(clean_text)
+    return len(alphabetic_chars) / len(clean_text)
+
+
+class HTMLTextExtractor(HTMLParser):
+    _thread_local = threading.local()  # 线程局部存储
+
+    def __init__(self):
+        super().__init__()
+        self.reset_state()
+
+    def handle_starttag(self, tag, attrs):
+        if tag in ('script', 'style'):
+            self.skip = True
+
+    def handle_endtag(self, tag):
+        if tag in ('script', 'style'):
+            self.skip = False
+
+    def handle_data(self, data):
+        if not self.skip and data.strip():
+            self.text.append(data)
+
+    def reset_state(self):
+        self.reset()
+        self.text = []
+        self.skip = False
+
+    def get_text(self):
+        return ''.join(self.text).strip()
+
+    @classmethod
+    def get_parser(cls):
+        # 每个线程获取独立实例
+        if not hasattr(cls._thread_local, 'parser'):
+            cls._thread_local.parser = cls()
+        return cls._thread_local.parser
+
+
+def clean_html(html):
+    parser = HTMLTextExtractor.get_parser()
+    parser.reset_state()
+    parser.feed(html)
+    parser.close()
+    return parser.get_text()
+
+
+

{re_common-10.0.2 → re_common-10.0.4}/re_common.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.2
+Version: 10.0.4
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.2 → re_common-10.0.4}/setup.py

@@ -24,8 +24,7 @@ re_common
   # new 20250224
   password_new = pypi-AgEIcHlwaS5vcmcCJDJkMzJhNWZkLTBkOTktNGE4Yi1iN2Q5LWU0ZjNjMGNiNzhjYwACEVsxLFsicmUtY29tbW9uIl1dAAIsWzIsWyIwYWIzYjIzMy0zZTlmLTQ1ZjAtYWQ5OC03NjJhNDM5NjNiZmUiXV0AAAYg0Pu6BqqupvhX7ad69d8e6XS8BmnLav6x6D9HJ-U3gko
   
-  pypi-AgEIcHlwaS5vcmcCJDUzNWEzYmFkLTcyMTUtNDE5OS05MGY4LWZlNTZkMDMzM2U2MQACKlszLCJhMjk4YTg3ZS02YjU2LTQzOTMtYjA5YS1kMzEzMzhjZmNiMDYiXQAABiD00KOVAf9T8AyK7F9K1G9LECXl0MdYa3yyODA7ox5qDg
-  
+ pypi-AgEIcHlwaS5vcmcCJDBlMzc1OTU1LWVkOGItNDVjZS1iMGNhLTQwYmNmYmU3NmViMwACEVsxLFsicmUtY29tbW9uIl1dAAIsWzIsWyIwYWIzYjIzMy0zZTlmLTQ1ZjAtYWQ5OC03NjJhNDM5NjNiZmUiXV0AAAYgisva56cDT5J3a9EBb3paplusibupSRSfoZc_L8PUGDY
 python setup.py sdist bdist_wheel
 python -m twine upload dist/*
 
@@ -35,7 +34,7 @@ long_description = """
 """
 setuptools.setup(
     name="re_common",
-    version="10.0.2",
+    version="10.0.4",
     author="vic",
     author_email="xujiang5@163.com",
     description="a library about all python projects",