re-common 10.0.3__py3-none-any.whl → 10.0.4__py3-none-any.whl

re_common/v2/baselibrary/utils/string_clear.py

@@ -1,7 +1,7 @@
 import re
 import regex
 
-from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant
+from re_common.v2.baselibrary.utils.stringutils import qj2bj, bj2qj, get_diacritic_variant, clean_html
 
 
 class StringClear(object):
@@ -88,20 +88,9 @@ class StringClear(object):
 
     def remove_html_tag(self):
         import html
-        from parsel import Selector
 
         self.obj_str = html.unescape(self.obj_str)
 
-        def clean_html(html):
-            sel = Selector(text=html, type='html')
-            # 移除脚本和样式
-            sel.xpath('//script').remove()
-            sel.xpath('//style').remove()
-
-            # 提取文本
-            text_nodes = sel.xpath('//text()').getall()
-            return ''.join(t.strip() for t in text_nodes if t.strip())
-
         self.obj_str = clean_html(self.obj_str)
 
         return self

re_common/v2/baselibrary/utils/stringutils.py

@@ -1,7 +1,9 @@
 import re
+import threading
 
 import regex
 import unicodedata
+from html.parser import HTMLParser
 
 
 def bj2qj(src):
@@ -25,6 +27,7 @@ def bj2qj(src):
 
     return ''.join(buf)
 
+
 def qj2bj(src):
     """
     全角转半角
@@ -62,6 +65,7 @@ def get_diacritic_variant(char1):
     # 判断基本字符是否相同
     return base_char1
 
+
 def get_alphabetic_ratio(text: str) -> float:
     # 返回字母型字符所占比例
     if not text:
@@ -93,4 +97,50 @@ def get_alphabetic_ratio(text: str) -> float:
     alphabetic_chars = re.findall(alphabetic_pattern, clean_text)
 
     # 返回字母型字符所占比例
-    return len(alphabetic_chars) / len(clean_text)
+    return len(alphabetic_chars) / len(clean_text)
+
+
+class HTMLTextExtractor(HTMLParser):
+    _thread_local = threading.local()  # 线程局部存储
+
+    def __init__(self):
+        super().__init__()
+        self.reset_state()
+
+    def handle_starttag(self, tag, attrs):
+        if tag in ('script', 'style'):
+            self.skip = True
+
+    def handle_endtag(self, tag):
+        if tag in ('script', 'style'):
+            self.skip = False
+
+    def handle_data(self, data):
+        if not self.skip and data.strip():
+            self.text.append(data)
+
+    def reset_state(self):
+        self.reset()
+        self.text = []
+        self.skip = False
+
+    def get_text(self):
+        return ''.join(self.text).strip()
+
+    @classmethod
+    def get_parser(cls):
+        # 每个线程获取独立实例
+        if not hasattr(cls._thread_local, 'parser'):
+            cls._thread_local.parser = cls()
+        return cls._thread_local.parser
+
+
+def clean_html(html):
+    parser = HTMLTextExtractor.get_parser()
+    parser.reset_state()
+    parser.feed(html)
+    parser.close()
+    return parser.get_text()
+
+
+

{re_common-10.0.3.dist-info → re_common-10.0.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.3
+Version: 10.0.4
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.3.dist-info → re_common-10.0.4.dist-info}/RECORD

@@ -178,8 +178,8 @@ re_common/v2/baselibrary/utils/basedict.py,sha256=tSV85pARe8ZQDY77_h_heS81EWwcgJ
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=NVV5Q0OMPlM_zTrs9ZDoPJv29GQv5wi9-AP1us5dBrQ,4651
 re_common/v2/baselibrary/utils/json_cls.py,sha256=dHOkWafG9lbQDoub9cbDwT2fDjMKtblQnjFLeA4hECA,286
 re_common/v2/baselibrary/utils/string_bool.py,sha256=4VCr1g8pX5YnzZSKctQgQfmhSQ0aw7a8ruhWdiRmBFU,641
-re_common/v2/baselibrary/utils/string_clear.py,sha256=okoBZCL1FytdI9FO7MBLTe1fvgGWt0-z87oUkg7jWkg,4171
-re_common/v2/baselibrary/utils/stringutils.py,sha256=quAgCdW_ayQwY4AqnZZkZ4NlcSEcy6f1arOVSeP2vEo,2699
+re_common/v2/baselibrary/utils/string_clear.py,sha256=6mkBAZUNh5-JTPmB9lj_i4eLT9C6ZW1nH4tZiGveIE4,3778
+re_common/v2/baselibrary/utils/stringutils.py,sha256=GLXHAm8IulC_8hWrN2aiFQjsoOpjczvcVozmTJj86-A,3864
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
 re_common/vip/baseencodeid.py,sha256=nERoe89ueFM52bG7xwJdflcZHk6T2RQQKbc5uUZc3RM,3272
@@ -206,8 +206,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.3.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.3.dist-info/METADATA,sha256=L9pU8jkMSaBnkTrRwU5xP9O5l5TGPQms5DZM-Ux4MDw,581
-re_common-10.0.3.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.3.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.3.dist-info/RECORD,,
+re_common-10.0.4.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.4.dist-info/METADATA,sha256=SNPQXc5koTrhSxu9yAPRPN42uItn6onNvmG7GHTMdcE,581
+re_common-10.0.4.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.4.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.4.dist-info/RECORD,,