re-common 10.0.39__py3-none-any.whl → 10.0.41__py3-none-any.whl

re_common/v2/baselibrary/tools/hdfs_line_processor.py

@@ -1,74 +1,74 @@
-import asyncio
-
-from pathlib import Path
-import time
-from typing import Any, Awaitable, Callable
-from re_common.v2.baselibrary.tools.concurrency import AsyncTaskPool
-from re_common.v2.baselibrary.tools.hdfs_base_processor import HDFSBaseProcessor
-
-
-class HDFSLineProcessor(HDFSBaseProcessor):
-    async def _process_data(self, data, process_func, pool):
-        """处理数据并执行处理函数"""
-        retry_count = 0
-        while retry_count < self.retries:
-            try:
-                return await process_func(data, pool)  # 成功处理后退出
-            except Exception as e:
-                retry_count += 1
-                print(f"处理数据时发生错误: {e}, 正在重试 {retry_count}/{self.retries}, data: {data}")
-                await asyncio.sleep(2**retry_count)
-        raise Exception(f"处理数据失败, 达到重试上限, data: {data}")
-
-    async def _process_file(self, hdfs_file_path, process_func):
-        """处理单个 gz 文件"""
-        start_time = time.perf_counter()
-        total_lines = self._count_total_lines(hdfs_file_path)
-        processed_lines = 0
-        pool = await self._get_pool()
-        results = []
-
-        for lines in self._batch_read_gz(hdfs_file_path):
-            processing_start_time = time.perf_counter()  # 记录本批处理开始时间
-
-            tasks = [self._process_data(line, process_func, pool) for line in lines]
-            results.extend(await AsyncTaskPool(self.concurrency).run(tasks))
-
-            processed_lines += len(lines)
-
-            self._print_progress(hdfs_file_path, processed_lines, total_lines, start_time, processing_start_time)
-
-        if self.result_dir is not None:
-            self.client.write(
-                self.result_dir.rstrip("/") + f"/{Path(hdfs_file_path).stem}",
-                data=self._generate_write_data(results),
-                overwrite=True,
-                encoding=self.encoding,
-            )
-
-        # 最终进度显示
-        self._print_final_progress(hdfs_file_path, processed_lines, total_lines, start_time)
-
-    async def map(self, process_func: Callable[[str, Any], Awaitable[Any]]) -> None:
-        gz_files = self._list_gz_files()
-        await self._run_multi_process(gz_files, process_func)
-
-
-# async def test_func(line: str, pool):
-#     pass
-
-
-# async def main():
-#     await HDFSLineProcessor(
-#         "/xx/xx",
-#         db_path=Path(__file__).parent / "test.db",
-#         concurrency=200,
-#         batch_size=1000,
-#         pool_factory=get_pool,
-#         max_processes=2,
-#         result_dir="/xx/xx_res",
-#     ).map(test_func)
-
-
-# if __name__ == "__main__":
-#     asyncio.run(main())
+import asyncio
+
+from pathlib import Path
+import time
+from typing import Any, Awaitable, Callable
+from re_common.v2.baselibrary.tools.concurrency import AsyncTaskPool
+from re_common.v2.baselibrary.tools.hdfs_base_processor import HDFSBaseProcessor
+
+
+class HDFSLineProcessor(HDFSBaseProcessor):
+    async def _process_data(self, data, process_func, pool):
+        """处理数据并执行处理函数"""
+        retry_count = 0
+        while retry_count < self.retries:
+            try:
+                return await process_func(data, pool)  # 成功处理后退出
+            except Exception as e:
+                retry_count += 1
+                print(f"处理数据时发生错误: {e}, 正在重试 {retry_count}/{self.retries}, data: {data}")
+                await asyncio.sleep(2**retry_count)
+        raise Exception(f"处理数据失败, 达到重试上限, data: {data}")
+
+    async def _process_file(self, hdfs_file_path, process_func):
+        """处理单个 gz 文件"""
+        start_time = time.perf_counter()
+        total_lines = self._count_total_lines(hdfs_file_path)
+        processed_lines = 0
+        pool = await self._get_pool()
+        results = []
+
+        for lines in self._batch_read_gz(hdfs_file_path):
+            processing_start_time = time.perf_counter()  # 记录本批处理开始时间
+
+            tasks = [self._process_data(line, process_func, pool) for line in lines]
+            results.extend(await AsyncTaskPool(self.concurrency).run(tasks))
+
+            processed_lines += len(lines)
+
+            self._print_progress(hdfs_file_path, processed_lines, total_lines, start_time, processing_start_time)
+
+        if self.result_dir is not None:
+            self.client.write(
+                self.result_dir.rstrip("/") + f"/{Path(hdfs_file_path).stem}",
+                data=self._generate_write_data(results),
+                overwrite=True,
+                encoding=self.encoding,
+            )
+
+        # 最终进度显示
+        self._print_final_progress(hdfs_file_path, processed_lines, total_lines, start_time)
+
+    async def map(self, process_func: Callable[[str, Any], Awaitable[Any]]) -> None:
+        gz_files = self._list_gz_files()
+        await self._run_multi_process(gz_files, process_func)
+
+
+# async def test_func(line: str, pool):
+#     pass
+
+
+# async def main():
+#     await HDFSLineProcessor(
+#         "/xx/xx",
+#         db_path=Path(__file__).parent / "test.db",
+#         concurrency=200,
+#         batch_size=1000,
+#         pool_factory=get_pool,
+#         max_processes=2,
+#         result_dir="/xx/xx_res",
+#     ).map(test_func)
+
+
+# if __name__ == "__main__":
+#     asyncio.run(main())

re_common/v2/baselibrary/tools/list_tools.py

@@ -1,70 +1,70 @@
-import itertools
-from typing import List, Any, Tuple
-
-
-def check_no_duplicates_2d(lst_2d):
-    """
-        检查二维列表的每一行是否无重复
-        如果有重复值 返回 False
-        如果没有重复 返回True
-    """
-    for row in lst_2d:
-        # 将行转为集合，比较长度
-        if len(row) != len(set(row)):
-            return False
-    return True
-
-
-def generate_cross_list_combinations(lists: List[List[Any]]) -> List[Tuple[Any, Any]]:
-    """
-    生成不同列表间的所有两两组合（元组长度为2）
-
-    参数:
-        lists: 包含多个列表的列表，例如 [[1,2], ['a','b'], ['x','y']]
-
-    返回:
-        包含所有跨列表两两组合的列表，每个组合是一个元组
-        例如 [(1,'a'), (1,'b'), (2,'a'), ..., ('a','x'), ('a','y'), ...]
-    """
-    combinations = []
-    for i in range(len(lists)):
-        for j in range(i + 1, len(lists)):
-            combinations.extend(itertools.product(lists[i], lists[j]))
-    return combinations
-
-
-def filter_and_sort_by_smi(all_list, top_n=1000):
-
-    """
-    要求 list 里面第一个是比较大小的数据 第二个是实际数据
-    """
-
-    # 1. 去重：按 doc_id 去重，保留 smi 最大的记录
-    unique_dict = {}
-    for smi, doc_id in all_list:
-        if doc_id not in unique_dict or smi > unique_dict[doc_id][0]:
-            unique_dict[doc_id] = (smi, doc_id)
-
-    # 2. 转换为列表并排序
-    unique_list = sorted(unique_dict.values(), key=lambda x: x[0], reverse=True)
-
-    # 3. 取前 top_n 个
-    return unique_list[:top_n]
-
-
-def list_to_dict(list_data,key_name):
-    # 使用 defaultdict 来处理重复 id
-    from collections import defaultdict
-
-    dict_data = defaultdict(list)
-
-    for item in list_data:
-        dict_data[item[key_name]].append(item)
-
-    # 将 defaultdict 转换成普通字典
-    dict_data = dict(dict_data)
-    return dict_data
-
-def split_list_by_step(lst, step=100):
-    # 一维列表按照步长转换成二维列表
+import itertools
+from typing import List, Any, Tuple
+
+
+def check_no_duplicates_2d(lst_2d):
+    """
+        检查二维列表的每一行是否无重复
+        如果有重复值 返回 False
+        如果没有重复 返回True
+    """
+    for row in lst_2d:
+        # 将行转为集合，比较长度
+        if len(row) != len(set(row)):
+            return False
+    return True
+
+
+def generate_cross_list_combinations(lists: List[List[Any]]) -> List[Tuple[Any, Any]]:
+    """
+    生成不同列表间的所有两两组合（元组长度为2）
+
+    参数:
+        lists: 包含多个列表的列表，例如 [[1,2], ['a','b'], ['x','y']]
+
+    返回:
+        包含所有跨列表两两组合的列表，每个组合是一个元组
+        例如 [(1,'a'), (1,'b'), (2,'a'), ..., ('a','x'), ('a','y'), ...]
+    """
+    combinations = []
+    for i in range(len(lists)):
+        for j in range(i + 1, len(lists)):
+            combinations.extend(itertools.product(lists[i], lists[j]))
+    return combinations
+
+
+def filter_and_sort_by_smi(all_list, top_n=1000):
+
+    """
+    要求 list 里面第一个是比较大小的数据 第二个是实际数据
+    """
+
+    # 1. 去重：按 doc_id 去重，保留 smi 最大的记录
+    unique_dict = {}
+    for smi, doc_id in all_list:
+        if doc_id not in unique_dict or smi > unique_dict[doc_id][0]:
+            unique_dict[doc_id] = (smi, doc_id)
+
+    # 2. 转换为列表并排序
+    unique_list = sorted(unique_dict.values(), key=lambda x: x[0], reverse=True)
+
+    # 3. 取前 top_n 个
+    return unique_list[:top_n]
+
+
+def list_to_dict(list_data,key_name):
+    # 使用 defaultdict 来处理重复 id
+    from collections import defaultdict
+
+    dict_data = defaultdict(list)
+
+    for item in list_data:
+        dict_data[item[key_name]].append(item)
+
+    # 将 defaultdict 转换成普通字典
+    dict_data = dict(dict_data)
+    return dict_data
+
+def split_list_by_step(lst, step=100):
+    # 一维列表按照步长转换成二维列表
     return [lst[i:i + step] for i in range(0, len(lst), step)]

re_common/v2/baselibrary/tools/resume_tracker.py

@@ -1,94 +1,94 @@
-import logging
-from pathlib import Path
-import sqlite3
-from typing import Iterable, List, Literal, Union
-
-logger = logging.getLogger(__name__)
-
-
-class ResumeTracker:
-    def __init__(
-        self,
-        db_path: Union[str, Path] = "processed.db",
-        timeout: float = 10.0,
-        isolation_level: Union[Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"], None] = "DEFERRED",
-    ):
-        self.db_path = Path(db_path)
-        self.timeout = timeout
-        self.isolation_level = isolation_level
-        self.init_db()
-
-    def _get_connection(self) -> sqlite3.Connection:
-        """创建数据库连接"""
-        return sqlite3.connect(self.db_path, timeout=self.timeout, isolation_level=self.isolation_level)
-
-    def init_db(self):
-        with self._get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("""
-                    CREATE TABLE IF NOT EXISTS processed_items (
-                        item_key TEXT PRIMARY KEY
-                    )
-                    """)
-            conn.commit()
-
-    def is_processed(self, item_key: str) -> bool:
-        with self._get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute(
-                "SELECT 1 FROM processed_items WHERE item_key = ?",
-                (item_key,),
-            )
-            return cursor.fetchone() is not None
-
-    def mark_processed(self, item_key: str):
-        with self._get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute(
-                "INSERT OR IGNORE INTO processed_items (item_key) VALUES (?)",
-                (item_key,),
-            )
-            conn.commit()
-
-    def mark_many_processed(self, item_keys: Iterable[str]):
-        with self._get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.executemany(
-                "INSERT OR IGNORE INTO processed_items (item_key) VALUES (?)",
-                [(key,) for key in item_keys],
-            )
-            conn.commit()
-
-    def get_processed_count(self) -> int:
-        with self._get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("SELECT COUNT(*) FROM processed_items")
-            return cursor.fetchone()[0]
-
-    def get_processed_items(self) -> List[str]:
-        with self._get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("SELECT item_key FROM processed_items")
-            return [row[0] for row in cursor.fetchall()]
-
-    def clear_processed_items(self):
-        with self._get_connection() as conn:
-            conn.execute("DELETE FROM processed_items")
-            logger.info("Cleared all processed items")
-
-
-if __name__ == "__main__":
-    tracker = ResumeTracker()
-    # 测试标记功能
-    tracker.mark_processed("test_key")
-    print(f"Is 'test_key' processed? {tracker.is_processed('test_key')}")
-
-    # 批量处理示例
-    test_keys = [f"key_{i}" for i in range(1, 10000)]
-    tracker.mark_many_processed(test_keys)
-
-    # 显示处理计数
-    print(f"Total processed items: {tracker.get_processed_count()}")
-
-    # 清理测试数据
-    tracker.clear_processed_items()
+import logging
+from pathlib import Path
+import sqlite3
+from typing import Iterable, List, Literal, Union
+
+logger = logging.getLogger(__name__)
+
+
+class ResumeTracker:
+    def __init__(
+        self,
+        db_path: Union[str, Path] = "processed.db",
+        timeout: float = 10.0,
+        isolation_level: Union[Literal["DEFERRED", "EXCLUSIVE", "IMMEDIATE"], None] = "DEFERRED",
+    ):
+        self.db_path = Path(db_path)
+        self.timeout = timeout
+        self.isolation_level = isolation_level
+        self.init_db()
+
+    def _get_connection(self) -> sqlite3.Connection:
+        """创建数据库连接"""
+        return sqlite3.connect(self.db_path, timeout=self.timeout, isolation_level=self.isolation_level)
+
+    def init_db(self):
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                    CREATE TABLE IF NOT EXISTS processed_items (
+                        item_key TEXT PRIMARY KEY
+                    )
+                    """)
+            conn.commit()
+
+    def is_processed(self, item_key: str) -> bool:
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                "SELECT 1 FROM processed_items WHERE item_key = ?",
+                (item_key,),
+            )
+            return cursor.fetchone() is not None
+
+    def mark_processed(self, item_key: str):
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                "INSERT OR IGNORE INTO processed_items (item_key) VALUES (?)",
+                (item_key,),
+            )
+            conn.commit()
+
+    def mark_many_processed(self, item_keys: Iterable[str]):
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.executemany(
+                "INSERT OR IGNORE INTO processed_items (item_key) VALUES (?)",
+                [(key,) for key in item_keys],
+            )
+            conn.commit()
+
+    def get_processed_count(self) -> int:
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT COUNT(*) FROM processed_items")
+            return cursor.fetchone()[0]
+
+    def get_processed_items(self) -> List[str]:
+        with self._get_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT item_key FROM processed_items")
+            return [row[0] for row in cursor.fetchall()]
+
+    def clear_processed_items(self):
+        with self._get_connection() as conn:
+            conn.execute("DELETE FROM processed_items")
+            logger.info("Cleared all processed items")
+
+
+if __name__ == "__main__":
+    tracker = ResumeTracker()
+    # 测试标记功能
+    tracker.mark_processed("test_key")
+    print(f"Is 'test_key' processed? {tracker.is_processed('test_key')}")
+
+    # 批量处理示例
+    test_keys = [f"key_{i}" for i in range(1, 10000)]
+    tracker.mark_many_processed(test_keys)
+
+    # 显示处理计数
+    print(f"Total processed items: {tracker.get_processed_count()}")
+
+    # 清理测试数据
+    tracker.clear_processed_items()

re_common/v2/baselibrary/tools/search_hash_tools.py

@@ -1,54 +1,54 @@
-from typing import List
-
-import jieba
-from datasketch import MinHash, minhash
-
-from re_common.v2.baselibrary.decorators.utils import deprecated
-from re_common.v2.baselibrary.utils.string_bool import is_single_cjk_char
-
-@deprecated("请使用 TextMatcherV2 中的方法代替。")
-def tokenize(text: str, stopwords=None) -> List[str]:
-    """
-    分词并移除停用词
-    """
-    if stopwords is None:
-        stopwords = []
-    words = jieba.lcut(text)
-
-    # 统计单字符数据 长度，防止结巴分词分不了的单词 将数据分为单个字符
-
-    # 这里为什么使用函数 而不是在推导式中兼容，主要是在一些 spark中 推导式的if 条件不遵循最短路径原则会将表达式当做一个整体算子
-    def is_singel_en(i):
-        if len(i) == 1 and not is_single_cjk_char(i):
-            return True
-        return False
-
-    one_char_size = len([i for i in words if is_singel_en(i)])
-    all_size = len(words)
-    # 如果单字符个数超过一定比例 就直接用空格分词
-    if all_size != 0 and one_char_size / all_size > 0.6:
-        words = [i for i in text.split() if i.strip()]
-
-    # 过滤停用词和空字符
-    words = [w for w in words if w not in stopwords and w.strip()]
-    return words
-
-@deprecated("请使用 TextMatcherV2 中的方法代替。")
-def create_minhash(words: List[str], num_perm=128) -> MinHash:
-    """
-    为分词结果创建 MinHash
-    """
-    minhash = MinHash(num_perm=num_perm)
-    for word in words:
-        minhash.update(word.encode("utf-8"))
-    return minhash
-
-@deprecated("请使用 TextMatcherV2 中的方法代替。")
-def get_str_minhash(title):
-    from re_common.v2.baselibrary.utils.string_clear import rel_clear
-    rel_title = rel_clear(title)
-    if not rel_title:
-        return ""
-    words = tokenize(rel_title)
-    minhash = create_minhash(words)
-    return minhash
+from typing import List
+
+import jieba
+from datasketch import MinHash, minhash
+
+from re_common.v2.baselibrary.decorators.utils import deprecated
+from re_common.v2.baselibrary.utils.string_bool import is_single_cjk_char
+
+@deprecated("请使用 TextMatcherV2 中的方法代替。")
+def tokenize(text: str, stopwords=None) -> List[str]:
+    """
+    分词并移除停用词
+    """
+    if stopwords is None:
+        stopwords = []
+    words = jieba.lcut(text)
+
+    # 统计单字符数据 长度，防止结巴分词分不了的单词 将数据分为单个字符
+
+    # 这里为什么使用函数 而不是在推导式中兼容，主要是在一些 spark中 推导式的if 条件不遵循最短路径原则会将表达式当做一个整体算子
+    def is_singel_en(i):
+        if len(i) == 1 and not is_single_cjk_char(i):
+            return True
+        return False
+
+    one_char_size = len([i for i in words if is_singel_en(i)])
+    all_size = len(words)
+    # 如果单字符个数超过一定比例 就直接用空格分词
+    if all_size != 0 and one_char_size / all_size > 0.6:
+        words = [i for i in text.split() if i.strip()]
+
+    # 过滤停用词和空字符
+    words = [w for w in words if w not in stopwords and w.strip()]
+    return words
+
+@deprecated("请使用 TextMatcherV2 中的方法代替。")
+def create_minhash(words: List[str], num_perm=128) -> MinHash:
+    """
+    为分词结果创建 MinHash
+    """
+    minhash = MinHash(num_perm=num_perm)
+    for word in words:
+        minhash.update(word.encode("utf-8"))
+    return minhash
+
+@deprecated("请使用 TextMatcherV2 中的方法代替。")
+def get_str_minhash(title):
+    from re_common.v2.baselibrary.utils.string_clear import rel_clear
+    rel_title = rel_clear(title)
+    if not rel_title:
+        return ""
+    words = tokenize(rel_title)
+    minhash = create_minhash(words)
+    return minhash