re-common 10.0.35__py3-none-any.whl → 10.0.37__py3-none-any.whl

re_common/v2/baselibrary/utils/db.py

@@ -1,4 +1,6 @@
 import os
+import time
+
 import aiomysql
 import asyncio
 from contextlib import asynccontextmanager
@@ -92,6 +94,7 @@ aiomysql_pool = None
 pool_lock = asyncio.Lock()  # 全局异步锁
 
 
+
 async def init_aiomysql_pool_async():
     global aiomysql_pool
     if aiomysql_pool is None:
@@ -100,3 +103,54 @@ async def init_aiomysql_pool_async():
                 print(f"[{os.getpid()}] Initializing aiomysql pool...")
                 aiomysql_pool = await aiomysql.create_pool(**DB_CONFIG)
     return aiomysql_pool
+
+
+motor_fs = None
+client = None
+motor_fs_lock = asyncio.Lock()  # 全局异步锁
+_loop_id_mongo = None
+
+async def check_connection(client):
+    try:
+        print("check mongodb client ping")
+        await client.admin.command("ping")
+        return True
+    except Exception:
+        return False
+
+
+async def init_motor_async(uri, db_name, bucket_name, is_reload=False):
+    global motor_fs, client, _loop_id_mongo
+    is_ping = True
+
+    if _loop_id_mongo is not None:
+        loop_id = id(asyncio.get_running_loop())
+        if loop_id != _loop_id_mongo:
+            is_reload = True
+
+    # 防止 每次都检查 只有 is_reload 时才检查连接
+    if is_reload:
+        is_ping = await check_connection(client)
+    if motor_fs is None or not is_ping:
+        async with motor_fs_lock:
+            if motor_fs is None or not is_ping:
+                print(f"[{os.getpid()}] Initializing motor_fs...")
+                from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorGridFSBucket
+                client = AsyncIOMotorClient(uri)
+                db = client[db_name]
+                motor_fs = AsyncIOMotorGridFSBucket(database=db, bucket_name=bucket_name)
+                _loop_id_mongo = id(asyncio.get_running_loop())
+    return motor_fs, client
+
+# async def run_main():
+#     while True:
+#         uri = "mongodb://192.168.98.80:27001/wpdc"
+#         db_name = "wpdc"
+#         bucket_name = "sci_doc"
+#         motor_fs, client = await init_motor_async(uri, db_name, bucket_name,is_reload=True)
+#         # print(await check_connection(client))
+#         time.sleep(3)
+#
+#
+# if __name__ == "__main__":
+#     asyncio.run(run_main())

re_common/v2/baselibrary/utils/stringutils.py

@@ -179,12 +179,22 @@ class HTMLTextExtractor(HTMLParser):
 #     parser.close()
 #     return parser.get_text()
 
+# def clean_html(html):
+#     """使用 Parsel 提取 HTML 中的纯文本"""
+#     sel = Selector(text=html, type='html')
+#     # 提取所有文本（包括子元素的文本）
+#     text = sel.xpath("string()").getall()
+#     return "".join(text).strip()
+
+
 def clean_html(html):
-    """使用 Parsel 提取 HTML 中的纯文本"""
-    sel = Selector(text=html)
-    # 提取所有文本（包括子元素的文本）
-    text = sel.xpath("string()").getall()
-    return "".join(text).strip()
+    from bs4 import BeautifulSoup
+    try:
+        soup = BeautifulSoup(html, "lxml")
+        return soup.get_text()
+    except:
+        soup = BeautifulSoup(html, "html5lib")
+        return soup.get_text()
 
 
 def remove_spaces_between_chinese_characters(text):

{re_common-10.0.35.dist-info → re_common-10.0.37.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.35
+Version: 10.0.37
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.35.dist-info → re_common-10.0.37.dist-info}/RECORD

@@ -207,14 +207,14 @@ re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZ
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=wwvk4kvipD-AVWCF0WRzBTM2q2wAvQtanLK-Hjp8FOk,6026
 re_common/v2/baselibrary/utils/basepika.py,sha256=ifOb3UsGj79k40aD9UK6-5BMPw43ZAo0SO3AYD4q4vw,7332
 re_common/v2/baselibrary/utils/basetime.py,sha256=b7U_ho6nE3fjYBxSkdMHXUOd3ClH6KkW_7p7l2Gs4gA,3038
-re_common/v2/baselibrary/utils/db.py,sha256=JktBem1IAsg7m-Wnsa-AT0Hj4bIkHyoKGBEDI1rL5Fc,3135
+re_common/v2/baselibrary/utils/db.py,sha256=5IOYOOGe6mNKhOeaCAq0iif5yJdDotBmrsMZmT1syjs,4862
 re_common/v2/baselibrary/utils/json_cls.py,sha256=M93piYtmgm_wP8E57culTrd_AhHLoGg6PqeAJYdW2SM,438
 re_common/v2/baselibrary/utils/mq.py,sha256=UHpO8iNIHs91Tgp-BgnSUpZwjWquxrGLdpr3FMMv2zw,2858
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
 re_common/v2/baselibrary/utils/string_bool.py,sha256=vxnjSFOfuHWGxkqaIbUNn21opx5tfV1uCXSahFfp1mU,6197
 re_common/v2/baselibrary/utils/string_clear.py,sha256=Ympa0Cs2y_72QeeyMS8de8y_QgtEFJJQ0AgHnylbMUc,7861
 re_common/v2/baselibrary/utils/string_smi.py,sha256=cU0WAWHRGnGoVQx3eCEKeM_q_olFNzRTJe7rSe586SY,741
-re_common/v2/baselibrary/utils/stringutils.py,sha256=jUmtyo4qu-GmE7_Y8gqFcYPJ7kOo8MSwxEsT7vGp9_c,8131
+re_common/v2/baselibrary/utils/stringutils.py,sha256=KnvKKmYafz-NbqnQq8iKXxQtA1xpHKaYzKcZex8yuM0,8397
 re_common/vip/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 re_common/vip/base_step_process.py,sha256=VXXiNj0I5CpzXIMCgOPU86bzDJkSBkUS-9CpZIl_GOk,205
 re_common/vip/baseencodeid.py,sha256=nERoe89ueFM52bG7xwJdflcZHk6T2RQQKbc5uUZc3RM,3272
@@ -241,8 +241,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.35.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.35.dist-info/METADATA,sha256=Uh894BDybODTMQemgCS-4hSIRVchegBTzUmBY7XZqRo,582
-re_common-10.0.35.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.35.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.35.dist-info/RECORD,,
+re_common-10.0.37.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.37.dist-info/METADATA,sha256=Yg4jmi3x2Z1aU5nkOIhjtsvzNI4ZnDp3YwdgBYu8IY8,582
+re_common-10.0.37.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.37.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.37.dist-info/RECORD,,