re-common 10.0.15__py3-none-any.whl → 10.0.16__py3-none-any.whl

re_common/v2/baselibrary/tools/hdfs_data_processer.py

@@ -93,7 +93,7 @@ class HDFSDataProcessor:
             with gzip.GzipFile(fileobj=BytesIO(compressed_data)) as gz_file:  # 解压缩
                 content = gz_file.read().decode(encoding)  # 解码为字符串
                 print(f"文件读取成功: {gz_file_path}")
-                lines = [i for i in content.splitlines() if i.strip()]
+                lines = [i for i in content.split("\n") if i.strip()]
                 result = [lines[i:i + self.batch_size] for i in range(0, len(lines), self.batch_size)]
                 return result
 

re_common/v2/baselibrary/tools/list_tools.py

@@ -1,9 +1,52 @@
+import itertools
+from typing import List, Any, Tuple
+
 
 def check_no_duplicates_2d(lst_2d):
-    """检查二维列表的每一行是否无重复"""
+    """
+        检查二维列表的每一行是否无重复
+        如果有重复值 返回 False
+        如果没有重复 返回True
+    """
     for row in lst_2d:
         # 将行转为集合，比较长度
         if len(row) != len(set(row)):
             return False
     return True
 
+
+def generate_cross_list_combinations(lists: List[List[Any]]) -> List[Tuple[Any, Any]]:
+    """
+    生成不同列表间的所有两两组合（元组长度为2）
+
+    参数:
+        lists: 包含多个列表的列表，例如 [[1,2], ['a','b'], ['x','y']]
+
+    返回:
+        包含所有跨列表两两组合的列表，每个组合是一个元组
+        例如 [(1,'a'), (1,'b'), (2,'a'), ..., ('a','x'), ('a','y'), ...]
+    """
+    combinations = []
+    for i in range(len(lists)):
+        for j in range(i + 1, len(lists)):
+            combinations.extend(itertools.product(lists[i], lists[j]))
+    return combinations
+
+
+def filter_and_sort_by_smi(all_list, top_n=1000):
+
+    """
+    要求 list 里面第一个是比较大小的数据 第二个是实际数据
+    """
+
+    # 1. 去重：按 doc_id 去重，保留 smi 最大的记录
+    unique_dict = {}
+    for smi, doc_id in all_list:
+        if doc_id not in unique_dict or smi > unique_dict[doc_id][0]:
+            unique_dict[doc_id] = (smi, doc_id)
+
+    # 2. 转换为列表并排序
+    unique_list = sorted(unique_dict.values(), key=lambda x: x[0], reverse=True)
+
+    # 3. 取前 top_n 个
+    return unique_list[:top_n]

re_common/v2/baselibrary/utils/db.py

@@ -2,7 +2,7 @@ from contextlib import asynccontextmanager
 from typing import AsyncGenerator, Tuple
 
 import aiomysql
-from aiomysql import Pool, Connection, Cursor
+from aiomysql import Pool, Connection, Cursor, DictCursor
 
 DB_CONFIG = {
     'host': '192.168.98.55',

re_common/v2/baselibrary/utils/string_bool.py

@@ -99,4 +99,51 @@ def is_all_symbols(text):
         return False
 
     # 检查每个字符是否属于符号类别
-    return all(unicodedata.category(char).startswith(('P', 'S')) for char in text)
+    return all(unicodedata.category(char).startswith(('P', 'S')) for char in text)
+
+
+def is_whole_word_en(sub_str: str, long_str: str) -> bool:
+    """
+        判断 sub_str 在 long_str 中是否是一个完整的单词（而不是其他单词的一部分）。
+
+        参数:
+            sub_str: 要搜索的单词
+            long_str: 被搜索的文本
+
+        返回:
+            bool: 如果 sub_str 是 long_str 中的一个完整单词则返回 True，否则返回 False
+        """
+    regex_pattern = re.compile(r"[^a-z0-9]")
+
+    if not sub_str or not long_str:
+        return False
+
+    # 使用 startsWith 和 endsWith 检查边界
+    if long_str.startswith(sub_str) and long_str.endswith(sub_str):
+        return True
+
+    # 检查是否在中间位置，且前后有非字母数字字符
+    index = long_str.find(sub_str)
+    if index >= 0:
+        if index == 0:
+            is_start = True
+        else:
+            is_start = bool(regex_pattern.match(long_str[index - 1]))
+
+        if len(long_str) == len(sub_str) + index:
+            is_end = True
+        else:
+            is_end = bool(regex_pattern.match(long_str[index + len(sub_str)]))
+
+        return is_start and is_end
+    else:
+        return False
+
+
+def is_whole_word(sub_str: str, long_str: str) -> bool:
+    if contains_chinese_chars(sub_str):
+        return True
+    elif is_whole_word_en(sub_str, long_str):
+        return True
+    else:
+        return False

{re_common-10.0.15.dist-info → re_common-10.0.16.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: re_common
-Version: 10.0.15
+Version: 10.0.16
 Summary: a library about all python projects
 Home-page: https://gitee.com/xujiangios/re-common
 Author: vic

{re_common-10.0.15.dist-info → re_common-10.0.16.dist-info}/RECORD

@@ -173,8 +173,8 @@ re_common/v2/baselibrary/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5
 re_common/v2/baselibrary/tools/ac_ahocorasick.py,sha256=c63y5RtKVLD37nyPCnBqfNygwRj4gTQqyIdDOrC65G0,2847
 re_common/v2/baselibrary/tools/dict_tools.py,sha256=BTh7oJuJ619IZgxiYlim0ltrXBclDtb7WzyFGr7wVf0,1246
 re_common/v2/baselibrary/tools/dolphinscheduler.py,sha256=1m7UGYDiuvJUCI6ik6CGM2fO8U5XteJzn55VRbwB9ts,7978
-re_common/v2/baselibrary/tools/hdfs_data_processer.py,sha256=cChy6vhK8uSVIf3bRMGWpjociIbkiV-0j29WlZqQXHM,14207
-re_common/v2/baselibrary/tools/list_tools.py,sha256=qYxdLccRbrULOBbaPdJ_MyFFmVJGVMdW5E36nJ3ejr8,249
+re_common/v2/baselibrary/tools/hdfs_data_processer.py,sha256=9V6EPkfefQEOnzia3xj2zhPBy_8nbzxFB88G1JjLyfc,14206
+re_common/v2/baselibrary/tools/list_tools.py,sha256=M4iT6HiKUPy3hTp5g-g7yV7-x-h0HVye7I3STU5A6Ww,1629
 re_common/v2/baselibrary/tools/search_hash_tools.py,sha256=2ENLtZE8opRsfkwRtTNMzITmpTsjO7wZ1ZkfkqpOH9U,1937
 re_common/v2/baselibrary/tools/text_matcher.py,sha256=cPMoFxaA0-ce3tLRxVSs8_3pTYS1oVIHDnNy_AlPU-4,10756
 re_common/v2/baselibrary/tools/unionfind_tools.py,sha256=VYHZZPXwBYljsm7TjV1B6iCgDn3O3btzNf9hMvQySVU,2965
@@ -184,11 +184,11 @@ re_common/v2/baselibrary/utils/author_smi.py,sha256=1ebH3AHv19jtJWdlqNdwu6t58HNV
 re_common/v2/baselibrary/utils/basedict.py,sha256=sH3_RZ8u4649-jX2V1uKNNkjJVUijZBDp6SdqncOZ88,1583
 re_common/v2/baselibrary/utils/basehdfs.py,sha256=NVV5Q0OMPlM_zTrs9ZDoPJv29GQv5wi9-AP1us5dBrQ,4651
 re_common/v2/baselibrary/utils/basepika.py,sha256=ifOb3UsGj79k40aD9UK6-5BMPw43ZAo0SO3AYD4q4vw,7332
-re_common/v2/baselibrary/utils/db.py,sha256=6HfmQHAtDm-pFFoe-ouNQggkfGRdN8Do2pN4B0ev_WU,1204
+re_common/v2/baselibrary/utils/db.py,sha256=lr6SI1Bk7tA0nw_4vHbmz-OaisPlb_p9ziC1Oz74tcA,1216
 re_common/v2/baselibrary/utils/json_cls.py,sha256=dHOkWafG9lbQDoub9cbDwT2fDjMKtblQnjFLeA4hECA,286
 re_common/v2/baselibrary/utils/mq.py,sha256=UHpO8iNIHs91Tgp-BgnSUpZwjWquxrGLdpr3FMMv2zw,2858
 re_common/v2/baselibrary/utils/n_ary_expression_tree.py,sha256=-05kO6G2Rth7CEK-5lfFrthFZ1Q0-0a7cni7mWZ-2gg,9172
-re_common/v2/baselibrary/utils/string_bool.py,sha256=0JxzftuL61UAF-2Vp9F1Og8kXp_y647KJC5jXus9QwM,3278
+re_common/v2/baselibrary/utils/string_bool.py,sha256=S5HMemxl2S248p4sEakD1NFJccH1NMQcbOFGmSFfcbg,4695
 re_common/v2/baselibrary/utils/string_clear.py,sha256=1QAb_IC8FoVL5KzXhPicz4stsYD7LyASh5sXaXfs084,6445
 re_common/v2/baselibrary/utils/string_smi.py,sha256=cU0WAWHRGnGoVQx3eCEKeM_q_olFNzRTJe7rSe586SY,741
 re_common/v2/baselibrary/utils/stringutils.py,sha256=WuxhXJVU6xuGfgHiSjxrn7Go1eobpa8DMR3Icoey4vo,6039
@@ -218,8 +218,8 @@ re_common/vip/title/transform/TransformRegulationTitleToZt.py,sha256=LKRdIsWKues
 re_common/vip/title/transform/TransformStandardTitleToZt.py,sha256=-fCKAbSBzXVyQDCE61CalvR9E_QzQMA08QOO_NePFNI,5563
 re_common/vip/title/transform/TransformThesisTitleToZt.py,sha256=QS-uV0cQrpUFAcKucuJQ9Ue2VRQH-inmfn_X3IplfRo,5488
 re_common/vip/title/transform/__init__.py,sha256=m83-CWyRq_VHPYHaALEQlmXrkTdrZ3e4B_kCfBYE-uc,239
-re_common-10.0.15.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-re_common-10.0.15.dist-info/METADATA,sha256=IhfGSUxRXpHVDZv-ZwqvSxD6yiI_WbeDEHVDpn-RvyU,582
-re_common-10.0.15.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-re_common-10.0.15.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
-re_common-10.0.15.dist-info/RECORD,,
+re_common-10.0.16.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+re_common-10.0.16.dist-info/METADATA,sha256=e3npLP7E8fdKFA2SnZe6qEAklRUJQOkJdhWPDfP8oKY,582
+re_common-10.0.16.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+re_common-10.0.16.dist-info/top_level.txt,sha256=_H9H23zoLIalm1AIY_KYTVh_H0ZnmjxQIxsvXtLv45o,10
+re_common-10.0.16.dist-info/RECORD,,